From 8eba2d7ea35853d75b62108a39dee75aff03df86 Mon Sep 17 00:00:00 2001 From: Juan RP Date: Tue, 17 Jul 2012 12:31:04 +0200 Subject: [PATCH] xbps-repo: replaced 'genindex' target with 'index-{add,clean}'. See the NEWS file for more information. --- NEWS | 12 + bin/xbps-repo/Makefile | 2 +- bin/xbps-repo/defs.h | 10 +- bin/xbps-repo/index-files.c | 588 ++++++++++++++++++++---------------- bin/xbps-repo/index-lock.c | 103 +++++++ bin/xbps-repo/index.c | 541 ++++++++++++++++----------------- bin/xbps-repo/main.c | 21 +- bin/xbps-repo/xbps-repo.8 | 15 +- include/xbps_api.h | 2 +- 9 files changed, 728 insertions(+), 566 deletions(-) create mode 100644 bin/xbps-repo/index-lock.c diff --git a/NEWS b/NEWS index 423ffaac..b06f74c3 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,15 @@ +xbps-0.16.6 (???): + + * xbps-repo(8): the 'genindex' target has been replaced by the 'index-add' and + 'index-clean' targets. The 'index-add' expects a list of binary packages to + be added to repository's index: + + $ xbps-repo index-add /path/to/repo/*.xbps + + The 'index-clean' removes obsolete entries in repository's index: + + $ xbps-repo index-clean /path/to/repo + xbps-0.16.5 (2012-07-14): * xbps.conf: remove obsolete remote repositories. diff --git a/bin/xbps-repo/Makefile b/bin/xbps-repo/Makefile index cb15c324..0948437b 100644 --- a/bin/xbps-repo/Makefile +++ b/bin/xbps-repo/Makefile @@ -3,7 +3,7 @@ TOPDIR = ../.. BIN = xbps-repo OBJS = main.o index.o show.o find-files.o list.o -OBJS += index-files.o clean.o +OBJS += index-files.o index-lock.o clean.o OBJS += ../xbps-bin/fetch_cb.o ../xbps-bin/util.o OBJS += ../xbps-bin/state_cb.o ../xbps-bin/list.o MAN = $(BIN).8 diff --git a/bin/xbps-repo/defs.h b/bin/xbps-repo/defs.h index 7213e406..a78d2316 100644 --- a/bin/xbps-repo/defs.h +++ b/bin/xbps-repo/defs.h @@ -35,10 +35,16 @@ struct repo_search_data { }; /* From index.c */ -int repo_genindex(struct xbps_handle *, const char *); +int repo_index_add(struct xbps_handle *, int, char **); +int repo_index_clean(struct xbps_handle *, const char *); /* From index-files.c */ -int repo_genindex_files(struct xbps_handle *, const char *); +int repo_index_files_add(struct xbps_handle *, int, char **); +int repo_index_files_clean(struct xbps_handle *, const char *); + +/* From index-common.c */ +int acquire_repo_lock(const char *, char **); +void release_repo_lock(char **, int); /* From find-files.c */ int repo_find_files_in_packages(struct xbps_handle *, int, char **); diff --git a/bin/xbps-repo/index-files.c b/bin/xbps-repo/index-files.c index a94435aa..9e2c2c9b 100644 --- a/bin/xbps-repo/index-files.c +++ b/bin/xbps-repo/index-files.c @@ -28,290 +28,348 @@ #include #include #include +#include +#include #include #include "defs.h" -struct index_files_data { - prop_array_t idx; - prop_array_t idxfiles; - prop_array_t obsoletes; - const char *pkgdir; - bool flush; - bool new; -}; - -static int -rmobsoletes_files_cb(struct xbps_handle *xhp, - prop_object_t obj, - void *arg, - bool *done) -{ - struct index_files_data *ifd = arg; - const char *pkgver, *arch; - char *str; - - (void)xhp; - (void)done; - - prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver); - prop_dictionary_get_cstring_nocopy(obj, "architecture", &arch); - if (xbps_find_pkg_in_array_by_pkgver(xhp, ifd->idx, pkgver, arch)) { - /* pkg found, do nothing */ - return 0; - } - if ((str = xbps_xasprintf("%s,%s", pkgver, arch)) == NULL) - return ENOMEM; - - if (!prop_array_add_cstring(ifd->obsoletes, str)) { - free(str); - return EINVAL; - } - free(str); - ifd->flush = true; - - return 0; -} - -static int -genindex_files_cb(struct xbps_handle *xhp, - prop_object_t obj, - void *arg, - bool *done) -{ - prop_object_t obj2, fileobj; - prop_dictionary_t pkg_filesd, pkgd; - prop_array_t files, pkg_cffiles, pkg_files, pkg_links; - struct index_files_data *ifd = arg; - const char *binpkg, *pkgver, *arch; - char *file; - bool found = false; - size_t i; - - (void)xhp; - (void)done; - - prop_dictionary_get_cstring_nocopy(obj, "filename", &binpkg); - prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver); - prop_dictionary_get_cstring_nocopy(obj, "architecture", &arch); - - if (xbps_find_pkg_in_array_by_pkgver(xhp, ifd->idxfiles, pkgver, arch)) { - fprintf(stderr, "index-files: skipping `%s' (%s), " - "already registered.\n", pkgver, arch); - return 0; - } - - file = xbps_xasprintf("%s/%s/%s", ifd->pkgdir, arch, binpkg); - if (file == NULL) - return ENOMEM; - - /* internalize files.plist from binary package archive */ - pkg_filesd = xbps_dictionary_metadata_plist_by_url(file, "./files.plist"); - if (pkg_filesd == NULL) { - free(file); - return EINVAL; - } - free(file); - - /* Find out if binary pkg stored in index contain any file */ - pkg_cffiles = prop_dictionary_get(pkg_filesd, "conf_files"); - if (pkg_cffiles != NULL && prop_array_count(pkg_cffiles)) - found = true; - else - pkg_cffiles = NULL; - - pkg_files = prop_dictionary_get(pkg_filesd, "files"); - if (pkg_files != NULL && prop_array_count(pkg_files)) - found = true; - else - pkg_files = NULL; - - pkg_links = prop_dictionary_get(pkg_filesd, "links"); - if (pkg_links != NULL && prop_array_count(pkg_links)) - found = true; - else - pkg_links = NULL; - - /* If pkg does not contain any file, ignore it */ - if (!found) { - prop_object_release(pkg_filesd); - return 0; - } - - /* create pkg dictionary */ - if ((pkgd = prop_dictionary_create()) == NULL) { - prop_object_release(pkg_filesd); - return ENOMEM; - } - /* add pkgver and architecture objects into pkg dictionary */ - if (!prop_dictionary_set_cstring(pkgd, "architecture", arch)) { - prop_object_release(pkg_filesd); - prop_object_release(pkgd); - return EINVAL; - } - if (!prop_dictionary_set_cstring(pkgd, "pkgver", pkgver)) { - prop_object_release(pkg_filesd); - prop_object_release(pkgd); - return EINVAL; - } - /* add files array obj into pkg dictionary */ - if ((files = prop_array_create()) == NULL) { - prop_object_release(pkg_filesd); - prop_object_release(pkgd); - return EINVAL; - } - if (!prop_dictionary_set(pkgd, "files", files)) { - prop_object_release(pkg_filesd); - prop_object_release(pkgd); - return EINVAL; - } - - /* add conf_files in pkgd */ - if (pkg_cffiles != NULL) { - for (i = 0; i < prop_array_count(pkg_cffiles); i++) { - obj2 = prop_array_get(pkg_cffiles, i); - fileobj = prop_dictionary_get(obj2, "file"); - if (!prop_array_add(files, fileobj)) { - prop_object_release(pkgd); - prop_object_release(pkg_filesd); - return EINVAL; - } - } - } - /* add files array in pkgd */ - if (pkg_files != NULL) { - for (i = 0; i < prop_array_count(pkg_files); i++) { - obj2 = prop_array_get(pkg_files, i); - fileobj = prop_dictionary_get(obj2, "file"); - if (!prop_array_add(files, fileobj)) { - prop_object_release(pkgd); - prop_object_release(pkg_filesd); - return EINVAL; - } - } - } - /* add links array in pkgd */ - if (pkg_links != NULL) { - for (i = 0; i < prop_array_count(pkg_links); i++) { - obj2 = prop_array_get(pkg_links, i); - fileobj = prop_dictionary_get(obj2, "file"); - if (!prop_array_add(files, fileobj)) { - prop_object_release(pkgd); - prop_object_release(pkg_filesd); - return EINVAL; - } - } - } - prop_object_release(pkg_filesd); - /* add pkgd into provided array */ - if (!prop_array_add(ifd->idxfiles, pkgd)) { - prop_object_release(pkgd); - return EINVAL; - } - printf("index-files: added `%s' (%s)\n", pkgver, arch); - prop_object_release(pkgd); - ifd->flush = true; - - return 0; -} - -/* - * Create the index files cache for all packages in repository. - */ int -repo_genindex_files(struct xbps_handle *xhp, const char *pkgdir) +repo_index_files_clean(struct xbps_handle *xhp, const char *repodir) { - prop_array_t idx; - struct index_files_data *ifd = NULL; - size_t i, x; - const char *p, *arch; - char *plist, *pkgver; - int rv; + prop_object_t obj; + prop_array_t idx, idxfiles, obsoletes; + char *plist, *plistf, *plistf_lock, *pkgver, *str; + const char *p, *arch, *ipkgver, *iarch; + size_t x, i; + int rv = 0, fdlock; + bool flush = false; - plist = xbps_pkg_index_plist(xhp, pkgdir); - if (plist == NULL) - return ENOMEM; + plist = plistf = plistf_lock = pkgver = str = NULL; + idx = idxfiles = obsoletes = NULL; - /* internalize repository index plist */ - idx = prop_array_internalize_from_zfile(plist); - if (idx == NULL) { - free(plist); - return errno; + /* Internalize index-files.plist if found */ + if ((plistf = xbps_pkg_index_files_plist(xhp, repodir)) == NULL) + return EINVAL; + if ((idxfiles = prop_array_internalize_from_zfile(plistf)) == NULL) { + free(plistf); + return 0; + } + /* Acquire exclusive file lock */ + if ((fdlock = acquire_repo_lock(plistf, &plistf_lock)) == -1) { + free(plistf); + prop_object_release(idxfiles); + return -1; } - free(plist); - /* internalize repository index-files plist (if exists) */ - plist = xbps_pkg_index_files_plist(xhp, pkgdir); - if (plist == NULL) { - rv = ENOMEM; + /* Internalize index.plist */ + if ((plist = xbps_pkg_index_plist(xhp, repodir)) == NULL) { + rv = EINVAL; goto out; } - ifd = calloc(1, sizeof(*ifd)); - if (ifd == NULL) { - rv = ENOMEM; + if ((idx = prop_array_internalize_from_zfile(plist)) == NULL) { + release_repo_lock(&plistf_lock, fdlock); + rv = EINVAL; goto out; } - ifd->pkgdir = pkgdir; - ifd->idxfiles = prop_array_internalize_from_zfile(plist); - ifd->idx = idx; - ifd->obsoletes = prop_array_create(); - if (ifd->idxfiles == NULL) { - /* missing file, create new one */ - ifd->idxfiles = prop_array_create(); - ifd->new = true; - } - - /* remove obsolete pkg entries */ - if (!ifd->new) { - rv = xbps_callback_array_iter(xhp, ifd->idxfiles, - rmobsoletes_files_cb, ifd); - if (rv != 0) - goto out; - for (i = 0; i < prop_array_count(ifd->obsoletes); i++) { - prop_array_get_cstring_nocopy(ifd->obsoletes, i, &p); - pkgver = strdup(p); - for (x = 0; x < strlen(p); x++) { - if ((pkgver[x] = p[x]) == ',') { - pkgver[x] = '\0'; - break; - } - } - arch = strchr(p, ',') + 1; - if (!xbps_remove_pkg_from_array_by_pkgver( - xhp, ifd->idxfiles, pkgver, arch)) { - free(pkgver); - rv = EINVAL; - goto out; - } - printf("index-files: removed obsolete entry `%s' " - "(%s)\n", pkgver, arch); - free(pkgver); + printf("Cleaning `%s' index-files, please wait...\n", repodir); + /* + * Iterate over index-files array to find obsolete entries. + */ + for (x = 0; x < prop_array_count(idx); x++) { + obj = prop_array_get(idx, x); + prop_dictionary_get_cstring_nocopy(obj, "pkgver", &ipkgver); + prop_dictionary_get_cstring_nocopy(obj, "architecture", &iarch); + if (xbps_find_pkg_in_array_by_pkgver(xhp, idx, ipkgver, iarch)) { + /* pkg found, do nothing */ + continue; } + if ((str = xbps_xasprintf("%s,%s", ipkgver, iarch)) == NULL) { + rv = ENOMEM; + goto out; + } + if (!prop_array_add_cstring(obsoletes, str)) { + free(str); + rv = EINVAL; + goto out; + } + free(str); } - /* iterate over index.plist array */ - if ((rv = xbps_callback_array_iter(xhp, idx, genindex_files_cb, ifd)) != 0) - goto out; - - if (!ifd->flush) - goto out; - - /* externalize index-files array */ - if (!prop_array_externalize_to_zfile(ifd->idxfiles, plist)) { + /* + * Iterate over the obsoletes and array and remove entries + * from index-files array. + */ + for (i = 0; i < prop_array_count(obsoletes); i++) { + prop_array_get_cstring_nocopy(obsoletes, i, &p); + pkgver = strdup(p); + for (x = 0; x < strlen(p); x++) { + if ((pkgver[x] = p[x]) == ',') { + pkgver[x] = '\0'; + break; + } + } + arch = strchr(p, ',') + 1; + if (!xbps_remove_pkg_from_array_by_pkgver( + xhp, idxfiles, pkgver, arch)) { + free(pkgver); + rv = EINVAL; + goto out; + } + printf("index-files: removed obsolete entry `%s' " + "(%s)\n", pkgver, arch); + free(pkgver); + flush = true; + } + /* Externalize index-files array to plist when necessary */ + if (flush && !prop_array_externalize_to_zfile(idxfiles, plistf)) rv = errno; - goto out; - } + + printf("index-files: %u packages registered.\n", + prop_array_count(idxfiles)); + out: - if (rv == 0) - printf("index-files: %u packages registered.\n", - prop_array_count(ifd->idxfiles)); - if (ifd->idxfiles != NULL) - prop_object_release(ifd->idxfiles); - if (plist != NULL) - free(plist); - if (ifd != NULL) - free(ifd); - if (idx != NULL) + release_repo_lock(&plistf_lock, fdlock); + + if (obsoletes) + prop_object_release(obsoletes); + if (idx) prop_object_release(idx); + if (idxfiles) + prop_object_release(idxfiles); + if (plist) + free(plist); + if (plistf) + free(plistf); + + return rv; +} + +int +repo_index_files_add(struct xbps_handle *xhp, int argc, char **argv) +{ + prop_array_t idxfiles = NULL; + prop_object_t obj, fileobj; + prop_dictionary_t pkgprops, pkg_filesd, pkgd; + prop_array_t files, pkg_cffiles, pkg_files, pkg_links; + const char *binpkg, *pkgver, *arch; + char *plist, *repodir, *p, *plist_lock; + size_t x; + int i, fdlock = -1, rv = 0; + bool found, flush; + + found = flush = false; + plist = plist_lock = repodir = p = NULL; + obj = fileobj = NULL; + pkgprops = pkg_filesd = pkgd = NULL; + files = NULL; + + if ((p = strdup(argv[1])) == NULL) { + rv = ENOMEM; + goto out; + } + repodir = dirname(p); + if ((plist = xbps_pkg_index_files_plist(xhp, repodir)) == NULL) { + rv = ENOMEM; + goto out; + } + /* Acquire exclusive file lock or wait for it. + */ + if ((fdlock = acquire_repo_lock(plist, &plist_lock)) == -1) { + free(p); + free(plist); + return -1; + } + /* + * Internalize index-files.plist if found and process argv. + */ + if ((idxfiles = prop_array_internalize_from_zfile(plist)) == NULL) { + if (errno == ENOENT) { + idxfiles = prop_array_create(); + assert(idxfiles); + } else { + rv = errno; + goto out; + } + } + + for (i = 1; i < argc; i++) { + found = false; + pkgprops = xbps_dictionary_metadata_plist_by_url(argv[i], + "./props.plist"); + if (pkgprops == NULL) { + fprintf(stderr, "index-files: cannot internalize " + "%s props.plist: %s\n", argv[i], strerror(errno)); + continue; + } + prop_dictionary_get_cstring_nocopy(pkgprops, + "filename", &binpkg); + prop_dictionary_get_cstring_nocopy(pkgprops, + "pkgver", &pkgver); + prop_dictionary_get_cstring_nocopy(pkgprops, + "architecture", &arch); + + if (xbps_find_pkg_in_array_by_pkgver(xhp, idxfiles, + pkgver, arch)) { + fprintf(stderr, "index-files: skipping `%s' (%s), " + "already registered.\n", pkgver, arch); + prop_object_release(pkgprops); + pkgprops = NULL; + continue; + } + + /* internalize files.plist from binary package archive */ + pkg_filesd = xbps_dictionary_metadata_plist_by_url(argv[i], + "./files.plist"); + if (pkg_filesd == NULL) { + prop_object_release(pkgprops); + rv = EINVAL; + goto out; + } + + /* Find out if binary pkg stored in index contain any file */ + pkg_cffiles = prop_dictionary_get(pkg_filesd, "conf_files"); + if (pkg_cffiles != NULL && prop_array_count(pkg_cffiles)) + found = true; + else + pkg_cffiles = NULL; + + pkg_files = prop_dictionary_get(pkg_filesd, "files"); + if (pkg_files != NULL && prop_array_count(pkg_files)) + found = true; + else + pkg_files = NULL; + + pkg_links = prop_dictionary_get(pkg_filesd, "links"); + if (pkg_links != NULL && prop_array_count(pkg_links)) + found = true; + else + pkg_links = NULL; + + /* If pkg does not contain any file, ignore it */ + if (!found) { + prop_object_release(pkgprops); + prop_object_release(pkg_filesd); + continue; + } + /* create pkg dictionary */ + if ((pkgd = prop_dictionary_create()) == NULL) { + prop_object_release(pkgprops); + prop_object_release(pkg_filesd); + rv = EINVAL; + goto out; + } + /* add pkgver and architecture objects into pkg dictionary */ + if (!prop_dictionary_set_cstring(pkgd, "architecture", arch)) { + prop_object_release(pkgprops); + prop_object_release(pkg_filesd); + prop_object_release(pkgd); + rv = EINVAL; + goto out; + } + if (!prop_dictionary_set_cstring(pkgd, "pkgver", pkgver)) { + prop_object_release(pkgprops); + prop_object_release(pkg_filesd); + prop_object_release(pkgd); + rv = EINVAL; + goto out; + } + /* add files array obj into pkg dictionary */ + if ((files = prop_array_create()) == NULL) { + prop_object_release(pkgprops); + prop_object_release(pkg_filesd); + prop_object_release(pkgd); + rv = EINVAL; + goto out; + } + if (!prop_dictionary_set(pkgd, "files", files)) { + prop_object_release(pkgprops); + prop_object_release(pkg_filesd); + prop_object_release(files); + prop_object_release(pkgd); + rv = EINVAL; + goto out; + } + /* add conf_files in pkgd */ + if (pkg_cffiles != NULL) { + for (x = 0; x < prop_array_count(pkg_cffiles); x++) { + obj = prop_array_get(pkg_cffiles, x); + fileobj = prop_dictionary_get(obj, "file"); + if (!prop_array_add(files, fileobj)) { + prop_object_release(pkgprops); + prop_object_release(pkg_filesd); + prop_object_release(files); + prop_object_release(pkgd); + rv = EINVAL; + goto out; + } + } + } + /* add files array in pkgd */ + if (pkg_files != NULL) { + for (x = 0; x < prop_array_count(pkg_files); x++) { + obj = prop_array_get(pkg_files, x); + fileobj = prop_dictionary_get(obj, "file"); + if (!prop_array_add(files, fileobj)) { + prop_object_release(pkgprops); + prop_object_release(pkg_filesd); + prop_object_release(files); + prop_object_release(pkgd); + rv = EINVAL; + goto out; + } + } + } + /* add links array in pkgd */ + if (pkg_links != NULL) { + for (x = 0; x < prop_array_count(pkg_links); x++) { + obj = prop_array_get(pkg_links, x); + fileobj = prop_dictionary_get(obj, "file"); + if (!prop_array_add(files, fileobj)) { + prop_object_release(pkgprops); + prop_object_release(pkg_filesd); + prop_object_release(files); + prop_object_release(pkgd); + rv = EINVAL; + goto out; + } + } + } + /* add pkgd into the index-files array */ + if (!prop_array_add(idxfiles, pkgd)) { + prop_object_release(pkgprops); + prop_object_release(pkg_filesd); + prop_object_release(files); + prop_object_release(pkgd); + rv = EINVAL; + goto out; + } + flush = true; + printf("index-files: added `%s' (%s)\n", pkgver, arch); + prop_object_release(pkgprops); + prop_object_release(pkg_filesd); + prop_object_release(files); + prop_object_release(pkgd); + pkgprops = pkg_filesd = pkgd = NULL; + files = NULL; + } + + if (flush && !prop_array_externalize_to_zfile(idxfiles, plist)) { + fprintf(stderr, "failed to externalize %s: %s\n", + plist, strerror(errno)); + rv = errno; + } + printf("index-files: %u packages registered.\n", + prop_array_count(idxfiles)); + +out: + release_repo_lock(&plist_lock, fdlock); + + if (p) + free(p); + if (plist) + free(plist); + if (idxfiles) + prop_object_release(idxfiles); return rv; } diff --git a/bin/xbps-repo/index-lock.c b/bin/xbps-repo/index-lock.c new file mode 100644 index 00000000..135508e1 --- /dev/null +++ b/bin/xbps-repo/index-lock.c @@ -0,0 +1,103 @@ +/*- + * Copyright (c) 2012 Juan Romero Pardines. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "defs.h" + +int +acquire_repo_lock(const char *plist, char **plist_lock) +{ + int try = 0, fd = -1; + + *plist_lock = xbps_xasprintf("%s.lock", plist); + assert(*plist_lock); + + fd = open(*plist_lock, O_RDWR); + if (fd == -1) { + if (errno == ENOENT) { + fd = creat(*plist_lock, 0640); + if (fd == -1) { + fprintf(stderr, "Failed to create " + "repository file lock: %s\n", + strerror(errno)); + return -1; + } + } else { + fprintf(stderr, "Failed to open repository " + "file lock: %s\n", strerror(errno)); + return -1; + } + } + /* + * Acquire the the exclusive file lock or wait until + * it's available. + */ +#define WAIT_SECONDS 30 + while (lockf(fd, F_TLOCK, 0) < 0) { + if (errno == EAGAIN || errno == EACCES) { + if (++try < WAIT_SECONDS) { + fprintf(stderr,"Repository index file " + "is busy! retrying in 5 sec...\n"); + sleep(5); + continue; + } + } + fprintf(stderr, "Failed to acquire repository " + "file lock in %d seconds!\n", WAIT_SECONDS); + close(fd); + return -1; + } + return fd; +} + +void +release_repo_lock(char **plist_lock, int fd) +{ + assert(*plist_lock); + + if (fd == -1) + return; + if (lockf(fd, F_ULOCK, 0) == -1) { + fprintf(stderr, "failed to unlock file lock: %s\n", + strerror(errno)); + close(fd); + exit(EXIT_FAILURE); + } + close(fd); + unlink(*plist_lock); + free(*plist_lock); +} diff --git a/bin/xbps-repo/index.c b/bin/xbps-repo/index.c index 427f1acd..22a46df5 100644 --- a/bin/xbps-repo/index.c +++ b/bin/xbps-repo/index.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2009-2012 Juan Romero Pardines. + * Copyright (c) 2012 Juan Romero Pardines. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -23,6 +23,7 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include #include #include @@ -31,50 +32,51 @@ #include #include #include -#include #include #include "defs.h" -#ifndef __arraycount -#define __arraycount(a) (sizeof(a) / sizeof(*a)) -#endif - -static const char *archs[] = { "noarch", "i686", "x86_64" }; - /* * Removes stalled pkg entries in repository's index.plist file, if any * binary package cannot be read (unavailable, not enough perms, etc). */ -static int -remove_missing_binpkg_entries(struct xbps_handle *xhp, const char *repodir) +int +repo_index_clean(struct xbps_handle *xhp, const char *repodir) { prop_array_t array; prop_dictionary_t pkgd; const char *filen, *pkgver, *arch; - char *binpkg, *plist; - size_t i; - int rv = 0; - bool found = false; + char *binpkg, *plist, *plist_lock; + size_t i, idx = 0; + int fdlock, rv = 0; + bool flush = false; - plist = xbps_pkg_index_plist(xhp, repodir); - if (plist == NULL) + if ((plist = xbps_pkg_index_plist(xhp, repodir)) == NULL) return -1; + if ((fdlock = acquire_repo_lock(plist, &plist_lock)) == -1) { + free(plist); + return -1; + } + array = prop_array_internalize_from_zfile(plist); if (array == NULL) { if (errno != ENOENT) { xbps_error_printf("xbps-repo: cannot read `%s': %s\n", plist, strerror(errno)); - exit(EXIT_FAILURE); + free(plist); + release_repo_lock(&plist_lock, fdlock); + return -1; } else { + release_repo_lock(&plist_lock, fdlock); free(plist); return 0; } } + printf("Cleaning `%s' index, please wait...\n", repodir); again: - for (i = 0; i < prop_array_count(array); i++) { + for (i = idx; i < prop_array_count(array); i++) { pkgd = prop_array_get(array, i); prop_dictionary_get_cstring_nocopy(pkgd, "pkgver", &pkgver); prop_dictionary_get_cstring_nocopy(pkgd, "filename", &filen); @@ -90,310 +92,281 @@ again: pkgver, arch); prop_array_remove(array, i); free(binpkg); - found = true; + flush = true; + idx = i; goto again; } free(binpkg); } - if (found && !prop_array_externalize_to_zfile(array, plist)) + if (flush && !prop_array_externalize_to_zfile(array, plist)) rv = errno; free(plist); + printf("index: %u packages registered.\n", prop_array_count(array)); + prop_object_release(array); + release_repo_lock(&plist_lock, fdlock); return rv; } -static prop_array_t -repoidx_get(struct xbps_handle *xhp, const char *pkgdir) +/* + * Adds a binary package into the index and removes old binary package + * and entry when it's necessary. + */ +int +repo_index_add(struct xbps_handle *xhp, int argc, char **argv) { - prop_array_t array; - char *plist; - int rv; - /* - * Remove entries in repositories index for unexistent - * packages, i.e dangling entries. - */ - if ((rv = remove_missing_binpkg_entries(xhp, pkgdir)) != 0) - return NULL; - - plist = xbps_pkg_index_plist(xhp, pkgdir); - if (plist == NULL) - return NULL; - - array = prop_array_internalize_from_zfile(plist); - free(plist); - if (array == NULL) - array = prop_array_create(); - - return array; -} - -static int -add_binpkg_to_index(struct xbps_handle *xhp, - prop_array_t idx, - const char *repodir, - const char *file) -{ - prop_dictionary_t newpkgd, curpkgd; + prop_array_t idx = NULL; + prop_dictionary_t newpkgd = NULL, curpkgd; struct stat st; const char *pkgname, *version, *regver, *oldfilen, *oldpkgver; const char *arch, *oldarch; - char *sha256, *filen, *tmpfilen, *oldfilepath, *buf; - int ret = 0, rv = 0; + char *sha256, *filen, *repodir, *oldfilepath, *buf; + char *tmpfilen = NULL, *tmprepodir = NULL, *plist = NULL; + char *plist_lock = NULL; + int i, ret = 0, rv = 0, fdlock = -1; + bool flush = false; - tmpfilen = strdup(file); - if (tmpfilen == NULL) - return errno; + if ((tmprepodir = strdup(argv[1])) == NULL) { + rv = ENOMEM; + goto out; + } + repodir = dirname(tmprepodir); - filen = basename(tmpfilen); - if (strcmp(tmpfilen, filen) == 0) { - rv = EINVAL; + /* Internalize plist file or create it if doesn't exist */ + if ((plist = xbps_pkg_index_plist(xhp, repodir)) == NULL) + return -1; + + /* Acquire exclusive file lock */ + if ((fdlock = acquire_repo_lock(plist, &plist_lock)) == -1) { + rv = fdlock; goto out; } - newpkgd = xbps_dictionary_metadata_plist_by_url(file, "./props.plist"); - if (newpkgd == NULL) { - xbps_error_printf("failed to read %s metadata for `%s'," - " skipping!\n", XBPS_PKGPROPS, file); - goto out; + if ((idx = prop_array_internalize_from_zfile(plist)) == NULL) { + if (errno != ENOENT) { + xbps_error_printf("xbps-repo: cannot read `%s': %s\n", + plist, strerror(errno)); + rv = -1; + goto out; + } else { + idx = prop_array_create(); + assert(idx); + } } - prop_dictionary_get_cstring_nocopy(newpkgd, "pkgname", &pkgname); - prop_dictionary_get_cstring_nocopy(newpkgd, "version", &version); - prop_dictionary_get_cstring_nocopy(newpkgd, "architecture", &arch); + /* - * Check if this package exists already in the index, but first - * checking the version. If current package version is greater - * than current registered package, update the index; otherwise - * pass to the next one. + * Process all packages specified in argv. */ - curpkgd = xbps_find_pkg_in_array_by_name(xhp, idx, pkgname, arch); - if (curpkgd == NULL) { - if (errno && errno != ENOENT) { - prop_object_release(newpkgd); - rv = errno; - goto out; - } - } else { - prop_dictionary_get_cstring_nocopy(curpkgd, "version", ®ver); - ret = xbps_cmpver(version, regver); - if (ret == 0) { - /* same version */ - fprintf(stderr, "index: skipping `%s-%s' (%s), `%s-%s' already " - "registered.\n", pkgname, version, - arch, pkgname, regver); - prop_object_release(newpkgd); - rv = EEXIST; - goto out; - } else if (ret == -1) { - /* idx version is greater, remove current binpkg */ - oldfilepath = xbps_xasprintf("%s/%s/%s", - repodir, arch, filen); - assert(oldfilepath != NULL); - if (remove(oldfilepath) == -1) { - rv = errno; - xbps_error_printf("failed to remove old binpkg " - "`%s': %s\n", oldfilepath, strerror(rv)); - free(oldfilepath); - prop_object_release(newpkgd); - goto out; - } - free(oldfilepath); - buf = xbps_xasprintf("`%s-%s' (%s)", pkgname, version, arch); - assert(buf != NULL); - prop_object_release(newpkgd); - printf("index: removed obsolete binpkg %s.\n", buf); - free(buf); - rv = EEXIST; - goto out; - } - /* current binpkg is greater than idx version */ - prop_dictionary_get_cstring_nocopy(curpkgd, - "filename", &oldfilen); - prop_dictionary_get_cstring_nocopy(curpkgd, - "pkgver", &oldpkgver); - prop_dictionary_get_cstring_nocopy(curpkgd, - "architecture", &oldarch); - - buf = strdup(oldpkgver); - if (buf == NULL) { - prop_object_release(newpkgd); + for (i = 1; i < argc; i++) { + if ((tmpfilen = strdup(argv[i])) == NULL) { rv = ENOMEM; goto out; } - oldfilepath = xbps_xasprintf("%s/%s/%s", - repodir, oldarch, oldfilen); - if (oldfilepath == NULL) { - rv = errno; - prop_object_release(newpkgd); - free(buf); - goto out; + filen = basename(tmpfilen); + /* + * Read metadata props plist dictionary from binary package. + */ + newpkgd = xbps_dictionary_metadata_plist_by_url(argv[i], + "./props.plist"); + if (newpkgd == NULL) { + xbps_error_printf("failed to read %s metadata for `%s'," + " skipping!\n", XBPS_PKGPROPS, argv[i]); + free(tmpfilen); + filen = NULL; + continue; } - if (remove(oldfilepath) == -1) { - rv = errno; - xbps_error_printf("failed to remove old " - "package file `%s': %s\n", oldfilepath, - strerror(errno)); - free(oldfilepath); - prop_object_release(newpkgd); - free(buf); - goto out; - } - free(oldfilepath); - if (!xbps_remove_pkg_from_array_by_pkgver(xhp, idx, - buf, oldarch)) { - xbps_error_printf("failed to remove `%s' " - "from plist index: %s\n", buf, strerror(errno)); - prop_object_release(newpkgd); - free(buf); - goto out; - } - printf("index: removed obsolete entry/binpkg `%s' " - "(%s).\n", buf, arch); - free(buf); - } - - /* - * We have the dictionary now, add the required - * objects for the index. - */ - if (!prop_dictionary_set_cstring(newpkgd, "filename", filen)) { - prop_object_release(newpkgd); - rv = errno; - goto out; - } - if ((sha256 = xbps_file_hash(file)) == NULL) { - prop_object_release(newpkgd); - rv = errno; - goto out; - } - if (!prop_dictionary_set_cstring(newpkgd, "filename-sha256", sha256)) { - prop_object_release(newpkgd); - free(sha256); - rv = errno; - goto out; - } - free(sha256); - if (stat(file, &st) == -1) { - prop_object_release(newpkgd); - rv = errno; - goto out; - } - if (!prop_dictionary_set_uint64(newpkgd, "filename-size", - (uint64_t)st.st_size)) { - prop_object_release(newpkgd); - rv = errno; - goto out; - } - /* - * Add dictionary into the index and update package count. - */ - if (!xbps_add_obj_to_array(idx, newpkgd)) { - rv = EINVAL; - goto out; - } - printf("index: added `%s-%s' (%s).\n", pkgname, version, arch); - -out: - if (tmpfilen) - free(tmpfilen); - - return rv; -} - -int -repo_genindex(struct xbps_handle *xhp, const char *pkgdir) -{ - prop_array_t idx = NULL; - struct dirent *dp; - DIR *dirp; - size_t i; - char *curdir; - char *binfile, *plist; - int rv = 0; - bool registered_newpkgs = false, foundpkg = false; - - /* - * Create or read existing package index plist file. - */ - idx = repoidx_get(xhp, pkgdir); - if (idx == NULL) - return errno; - - plist = xbps_pkg_index_plist(xhp, pkgdir); - if (plist == NULL) { - prop_object_release(idx); - return errno; - } - - for (i = 0; i < __arraycount(archs); i++) { - curdir = xbps_xasprintf("%s/%s", pkgdir, archs[i]); - assert(curdir != NULL); - - dirp = opendir(curdir); - if (dirp == NULL) { - if (errno == ENOENT) { - free(curdir); - continue; - } - xbps_error_printf("xbps-repo: cannot open `%s': %s\n", - curdir, strerror(errno)); - exit(EXIT_FAILURE); - } - while ((dp = readdir(dirp)) != NULL) { - if ((strcmp(dp->d_name, ".") == 0) || - (strcmp(dp->d_name, "..") == 0)) - continue; - /* Ignore unknown files */ - if (strstr(dp->d_name, ".xbps") == NULL) - continue; - - foundpkg = true; - binfile = xbps_xasprintf("%s/%s", curdir, dp->d_name); - if (binfile == NULL) { - (void)closedir(dirp); + prop_dictionary_get_cstring_nocopy(newpkgd, "pkgname", + &pkgname); + prop_dictionary_get_cstring_nocopy(newpkgd, "version", + &version); + prop_dictionary_get_cstring_nocopy(newpkgd, "architecture", + &arch); + /* + * Check if this package exists already in the index, but first + * checking the version. If current package version is greater + * than current registered package, update the index; otherwise + * pass to the next one. + */ + curpkgd = + xbps_find_pkg_in_array_by_name(xhp, idx, pkgname, arch); + if (curpkgd == NULL) { + if (errno && errno != ENOENT) { + prop_object_release(newpkgd); + free(tmpfilen); rv = errno; goto out; } - rv = add_binpkg_to_index(xhp, idx, pkgdir, binfile); - free(binfile); - if (rv == EEXIST) { - rv = 0; + } else { + prop_dictionary_get_cstring_nocopy(curpkgd, + "version", ®ver); + ret = xbps_cmpver(version, regver); + if (ret == 0) { + /* Same version */ + fprintf(stderr, "index: skipping `%s-%s' " + "(%s), `%s-%s' already registered.\n", + pkgname, version, arch, pkgname, regver); + prop_object_release(newpkgd); + free(tmpfilen); + newpkgd = NULL; + filen = NULL; continue; - } else if (rv != 0) { - (void)closedir(dirp); - free(curdir); + } else if (ret == -1) { + /* + * Index version is greater, remove current + * package. + */ + oldfilepath = xbps_xasprintf("%s/%s/%s", + repodir, arch, filen); + assert(oldfilepath != NULL); + if (remove(oldfilepath) == -1) { + rv = errno; + xbps_error_printf("failed to remove " + "old binpkg `%s': %s\n", + oldfilepath, strerror(rv)); + prop_object_release(newpkgd); + free(tmpfilen); + free(oldfilepath); + goto out; + } + free(oldfilepath); + buf = xbps_xasprintf("`%s-%s' (%s)", + pkgname, version, arch); + assert(buf != NULL); + printf("index: removed obsolete binpkg %s.\n", + buf); + free(buf); + prop_object_release(newpkgd); + free(tmpfilen); + newpkgd = NULL; + filen = NULL; + continue; + } + /* + * Current package version is greater than + * index version. + */ + prop_dictionary_get_cstring_nocopy(curpkgd, + "filename", &oldfilen); + prop_dictionary_get_cstring_nocopy(curpkgd, + "pkgver", &oldpkgver); + prop_dictionary_get_cstring_nocopy(curpkgd, + "architecture", &oldarch); + + if ((buf = strdup(oldpkgver)) == NULL) { + rv = ENOMEM; goto out; } - registered_newpkgs = true; + oldfilepath = xbps_xasprintf("%s/%s/%s", + repodir, oldarch, oldfilen); + if (oldfilepath == NULL) { + rv = errno; + free(buf); + prop_object_release(newpkgd); + free(tmpfilen); + goto out; + } + if (remove(oldfilepath) == -1) { + rv = errno; + xbps_error_printf("failed to remove old " + "package file `%s': %s\n", oldfilepath, + strerror(errno)); + free(oldfilepath); + free(buf); + prop_object_release(newpkgd); + free(tmpfilen); + goto out; + } + free(oldfilepath); + if (!xbps_remove_pkg_from_array_by_pkgver(xhp, idx, + buf, oldarch)) { + xbps_error_printf("failed to remove `%s' " + "from plist index: %s\n", buf, + strerror(errno)); + rv = errno; + free(buf); + prop_object_release(newpkgd); + free(tmpfilen); + goto out; + } + printf("index: removed obsolete entry/binpkg `%s' " + "(%s).\n", buf, arch); + free(buf); } - (void)closedir(dirp); - free(curdir); + /* + * We have the dictionary now, add the required + * objects for the index. + */ + if (!prop_dictionary_set_cstring(newpkgd, "filename", filen)) { + rv = errno; + prop_object_release(newpkgd); + free(tmpfilen); + goto out; + } + if ((sha256 = xbps_file_hash(argv[i])) == NULL) { + rv = errno; + prop_object_release(newpkgd); + free(tmpfilen); + goto out; + } + if (!prop_dictionary_set_cstring(newpkgd, "filename-sha256", + sha256)) { + free(sha256); + prop_object_release(newpkgd); + free(tmpfilen); + rv = errno; + goto out; + } + free(sha256); + if (stat(argv[i], &st) == -1) { + prop_object_release(newpkgd); + free(tmpfilen); + rv = errno; + goto out; + } + if (!prop_dictionary_set_uint64(newpkgd, "filename-size", + (uint64_t)st.st_size)) { + prop_object_release(newpkgd); + free(tmpfilen); + rv = errno; + goto out; + } + /* + * Add new pkg dictionary into the index. + */ + if (!prop_array_add(idx, newpkgd)) { + prop_object_release(newpkgd); + free(tmpfilen); + rv = EINVAL; + goto out; + } + flush = true; + printf("index: added `%s-%s' (%s).\n", pkgname, version, arch); + free(tmpfilen); + prop_object_release(newpkgd); + newpkgd = NULL; + sha256 = NULL; + filen = NULL; + oldfilen = oldarch = oldpkgver = NULL; + pkgname = version = arch = NULL; } - if (foundpkg == false) { - /* No packages were found in directory */ - rv = ENOENT; - } else { - /* - * Show total count registered packages. - */ - printf("index: %zu packages registered.\n", - (size_t)prop_array_count(idx)); - /* - * Don't write plist file if no packages were registered. - */ - if (registered_newpkgs == false) - goto out; - /* - * If any package was registered in package index, write - * plist file to storage. - */ - if (!prop_array_externalize_to_zfile(idx, plist)) - rv = errno; + if (flush && !prop_array_externalize_to_zfile(idx, plist)) { + xbps_error_printf("failed to externalize plist: %s\n", + strerror(errno)); + rv = errno; } + printf("index: %u packages registered.\n", prop_array_count(idx)); + out: - free(plist); - prop_object_release(idx); + release_repo_lock(&plist_lock, fdlock); + + if (tmprepodir) + free(tmprepodir); + if (plist) + free(plist); + if (idx) + prop_object_release(idx); return rv; } diff --git a/bin/xbps-repo/main.c b/bin/xbps-repo/main.c index 4b297c59..cb5654f2 100644 --- a/bin/xbps-repo/main.c +++ b/bin/xbps-repo/main.c @@ -260,14 +260,25 @@ main(int argc, char **argv) xbps_error_printf("xbps-repo: no repositories " "currently registered!\n"); } - } else if (strcasecmp(argv[0], "genindex") == 0) { - /* Generates a package repository index plist file. */ + } else if (strcasecmp(argv[0], "index-add") == 0) { + /* Registers a binary package into the repository's index. */ + if (argc < 2) + usage(true); + + if ((rv = repo_index_add(&xh, argc, argv)) != 0) + goto out; + if ((rv = repo_index_files_add(&xh, argc, argv)) != 0) + goto out; + + } else if (strcasecmp(argv[0], "index-clean") == 0) { + /* Removes obsolete pkg entries from index in a repository */ if (argc != 2) usage(true); - rv = repo_genindex(&xh, argv[1]); - if (rv == 0) - rv = repo_genindex_files(&xh, argv[1]); + if ((rv = repo_index_clean(&xh, argv[1])) != 0) + goto out; + + rv = repo_index_files_clean(&xh, argv[1]); } else if (strcasecmp(argv[0], "sync") == 0) { /* Syncs the pkg index for all registered remote repos */ diff --git a/bin/xbps-repo/xbps-repo.8 b/bin/xbps-repo/xbps-repo.8 index e99ea159..41f1f006 100644 --- a/bin/xbps-repo/xbps-repo.8 +++ b/bin/xbps-repo/xbps-repo.8 @@ -1,4 +1,4 @@ -.Dd June 5, 2012 +.Dd July 17, 2012 .Os Void GNU/Linux .Dt xbps-repo 8 .Sh NAME @@ -64,13 +64,12 @@ Prints the name of .Em package(s) matching the pattern on its file list by looking in all repositories index files. Multiple patterns can be specified as arguments. -.It Sy genindex Pa /path/to/local/repo -Generates the package index files for a local repository as specified in its argument. -It will look for archives with the -.Sy .xbps -extension and will only add them onto the index if version is newer than the one -currently stored. If a newer package is available, the old binary package will be -removed automatically. +.It Sy index-add Ar /path/to/local/repository/foo-1.0.arch.xbps +This will register the binary package into the local repository's index files, and remove +old entry and binary package if any old version exists. +Multiple binary packages can be specified. +.It Sy index-clean Ar /path/to/local/repository +This will remove any obsolete entry found in the local repository's index files. .It Sy list Lists all working repositories in repository pool. .It Sy pkg-list Op repository diff --git a/include/xbps_api.h b/include/xbps_api.h index 145edb0a..68c925cb 100644 --- a/include/xbps_api.h +++ b/include/xbps_api.h @@ -57,7 +57,7 @@ #define XBPS_PKGINDEX_VERSION "1.5" #define XBPS_API_VERSION "20120714" -#define XBPS_VERSION "0.16.5" +#define XBPS_VERSION "0.16.6" /** * @def XBPS_RELVER