emo/xbps
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Get rid of repodata index-files; the archive is now 8x smaller.

Browse Source 
See the NEWS file for more information.
This commit is contained in:
Juan RP 2014-11-13 17:09:43 +01:00
parent e1bdeb83a1
commit 3afb9d709d
12 changed files with 65 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
NEWS
View File

@ -1,5 +1,13 @@
xbps-0.42 (???):
* Get rid of repodata index-files, that contained all pkg files of a repository
in the archive. That file was so huge that incresed the repository archive
size by 8x. That means that `xbps-query -Ro` will now make N parallel connections
to the remote server to inspect the binary packages remotely; if the binary
package is in cachedir, no network connection to the remote server will be
established. This reduces the size of the main x86_64 repository (4000 packages)
to 700KB, compared to 4.5MB previously.
* xbps-create(8): fixed issue #64 "xbps-create(8) incorrect installed-size with hardlinks".
https://github.com/voidlinux/xbps/issues/64

47
bin/xbps-query/ownedby.c
View File

@ -119,30 +119,37 @@ ownedby_pkgdb_cb(struct xbps_handle *xhp,
static int
repo_match_cb(struct xbps_handle *xhp _unused,
repo_match_cb(struct xbps_handle *xhp,
xbps_object_t obj,
const char *key,
const char *key _unused,
void *arg,
bool *done _unused)
{
xbps_dictionary_t filesd;
xbps_array_t files_keys;
struct ffdata *ffd = arg;
const char *filestr;
regex_t regex;
const char *pkgver;
char *bfile;
for (unsigned int i = 0; i < xbps_array_count(obj); i++) {
xbps_array_get_cstring_nocopy(obj, i, &filestr);
if (ffd->regex) {
if (regcomp(&regex, ffd->pat, REG_EXTENDED|REG_NOSUB) != 0)
return errno;
if (regexec(&regex, filestr, 0, 0, 0) == 0) {
printf("%s: %s (%s)\n", key, filestr, ffd->repouri);
}
regfree(&regex);
} else {
if ((fnmatch(ffd->pat, filestr, FNM_PERIOD)) == 0)
printf("%s: %s (%s)\n", key, filestr, ffd->repouri);
xbps_dictionary_set_cstring_nocopy(obj, "repository", ffd->repouri);
xbps_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
bfile = xbps_repository_pkg_path(xhp, obj);
assert(bfile);
filesd = xbps_get_pkg_plist_from_binpkg(bfile, "./files.plist");
if (filesd == NULL) {
xbps_dbg_printf(xhp, "%s: couldn't fetch files.plist from %s: %s\n",
pkgver, bfile, strerror(errno));
return EINVAL;
}
files_keys = xbps_dictionary_all_keys(filesd);
for (unsigned int i = 0; i < xbps_array_count(files_keys); i++) {
match_files_by_pattern(filesd,
xbps_array_get(files_keys, i), ffd, pkgver);
}
xbps_object_release(files_keys);
xbps_object_release(filesd);
free(bfile);
return 0;
}
@ -154,13 +161,9 @@ repo_ownedby_cb(struct xbps_repo *repo, void *arg, bool *done _unused)
struct ffdata *ffd = arg;
int rv;
xbps_repo_open_idxfiles(repo);
if (repo->idxfiles == NULL)
return 0;
ffd->repouri = repo->uri;
allkeys = xbps_dictionary_all_keys(repo->idxfiles);
rv = xbps_array_foreach_cb(repo->xhp, allkeys, repo->idxfiles, repo_match_cb, ffd);
allkeys = xbps_dictionary_all_keys(repo->idx);
rv = xbps_array_foreach_cb_multi(repo->xhp, allkeys, repo->idx, repo_match_cb, ffd);
xbps_object_release(allkeys);
return rv;

2
bin/xbps-rindex/defs.h
View File

@ -81,6 +81,6 @@ int sign_repo(struct xbps_handle *, const char *, const char *,
/* From repoflush.c */
bool repodata_flush(struct xbps_handle *, const char *,
xbps_dictionary_t, xbps_dictionary_t, xbps_dictionary_t);
xbps_dictionary_t, xbps_dictionary_t);
#endif /* !_XBPS_RINDEX_DEFS_H_ */

82
bin/xbps-rindex/index-add.c
View File

@ -40,14 +40,12 @@
int
index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force)
{
xbps_array_t array, pkg_files, pkg_links, pkg_cffiles;
xbps_dictionary_t idx, idxmeta, idxfiles, binpkgd, pkg_filesd, curpkgd;
xbps_object_t obj, fileobj;
xbps_dictionary_t idx, idxmeta, binpkgd, curpkgd;
struct xbps_repo *repo = NULL;
struct stat st;
char *tmprepodir = NULL, *repodir = NULL;
int rv = 0, ret = 0;
bool flush = false, found = false;
bool flush = false;
assert(argv);
/*
@ -64,15 +62,12 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force
rv = -1;
goto out;
}
if (repo && repo->idx) {
xbps_repo_open_idxfiles(repo);
if (repo) {
idx = xbps_dictionary_copy(repo->idx);
idxmeta = xbps_dictionary_copy(repo->idxmeta);
idxfiles = xbps_dictionary_copy(repo->idxfiles);
} else {
idx = xbps_dictionary_create();
idxmeta = NULL;
idxfiles = xbps_dictionary_create();
}
/*
* Process all packages specified in argv.
@ -153,7 +148,6 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force
*/
printf("index: removed obsolete entry `%s' (%s).\n", opkgver, oarch);
xbps_dictionary_remove(idx, pkgname);
xbps_dictionary_remove(idxfiles, opkgver);
free(opkgver);
free(oarch);
}
@ -205,87 +199,19 @@ index_add(struct xbps_handle *xhp, int args, int argmax, char **argv, bool force
printf("index: added `%s' (%s).\n", pkgver, arch);
xbps_object_release(binpkgd);
free(pkgname);
/*
* Add new pkg dictionary into the index-files.
*/
found = false;
pkg_filesd = xbps_get_pkg_plist_from_binpkg(pkg, "./files.plist");
if (pkg_filesd == NULL) {
free(pkgver);
rv = EINVAL;
goto out;
}
pkg_cffiles = xbps_dictionary_get(pkg_filesd, "conf_files");
if (xbps_array_count(pkg_cffiles))
found = true;
else
pkg_cffiles = NULL;
pkg_files = xbps_dictionary_get(pkg_filesd, "files");
if (xbps_array_count(pkg_files))
found = true;
else
pkg_files = NULL;
pkg_links = xbps_dictionary_get(pkg_filesd, "links");
if (xbps_array_count(pkg_links))
found = true;
else
pkg_links = NULL;
/* If pkg does not contain any file, ignore it */
if (!found) {
xbps_object_release(pkg_filesd);
free(pkgver);
continue;
}
/* create pkg files array */
array = xbps_array_create();
assert(array);
/* add conf_files in pkg files array */
if (pkg_cffiles != NULL) {
for (unsigned int x = 0; x < xbps_array_count(pkg_cffiles); x++) {
obj = xbps_array_get(pkg_cffiles, x);
fileobj = xbps_dictionary_get(obj, "file");
xbps_array_add(array, fileobj);
}
}
/* add files array in pkg files array */
if (pkg_files != NULL) {
for (unsigned int x = 0; x < xbps_array_count(pkg_files); x++) {
obj = xbps_array_get(pkg_files, x);
fileobj = xbps_dictionary_get(obj, "file");
xbps_array_add(array, fileobj);
}
}
/* add links array in pkg files array */
if (pkg_links != NULL) {
for (unsigned int x = 0; x < xbps_array_count(pkg_links); x++) {
obj = xbps_array_get(pkg_links, x);
fileobj = xbps_dictionary_get(obj, "file");
xbps_array_add(array, fileobj);
}
}
/* add pkg files array into index-files */
xbps_dictionary_set(idxfiles, pkgver, array);
xbps_object_release(array);
xbps_object_release(pkg_filesd);
free(pkgver);
}
/*
* Generate repository data files.
*/
if (flush) {
if (!repodata_flush(xhp, repodir, idx, idxfiles, idxmeta)) {
if (!repodata_flush(xhp, repodir, idx, idxmeta)) {
fprintf(stderr, "%s: failed to write repodata: %s\n",
_XBPS_RINDEX, strerror(errno));
goto out;
}
}
printf("index: %u packages registered.\n", xbps_dictionary_count(idx));
printf("index-files: %u packages registered.\n", xbps_dictionary_count(idxfiles));
out:
if (repo)

59
bin/xbps-rindex/index-clean.c
View File

@ -85,32 +85,6 @@ idx_cleaner_cb(struct xbps_handle *xhp,
return 0;
}
static int
idxfiles_cleaner_cb(struct xbps_handle *xhp _unused, xbps_object_t obj _unused,
const char *key, void *arg, bool *done _unused)
{
xbps_dictionary_t pkg;
struct cbdata *cbd = arg;
char *pkgname;
const char *pkgver;
/* Find out entries on index-files that aren't registered on index */
if ((pkgname = xbps_pkg_name(key)) == NULL) {
xbps_dbg_printf(xhp, "%s: invalid entry found on index-files: %s\n", cbd->repourl, key);
return 0;
}
if ((pkg = xbps_dictionary_get(cbd->idx, pkgname))) {
xbps_dictionary_get_cstring_nocopy(pkg, "pkgver", &pkgver);
if (strcmp(pkgver, key)) {
pthread_mutex_lock(&cbd->mtx);
xbps_array_add_cstring_nocopy(cbd->result, key);
pthread_mutex_unlock(&cbd->mtx);
}
}
free(pkgname);
return 0;
}
/*
* Removes stalled pkg entries in repository's XBPS_REPOIDX file, if any
* binary package cannot be read (unavailable, not enough perms, etc).
@ -119,7 +93,7 @@ int
index_clean(struct xbps_handle *xhp, const char *repodir)
{
xbps_array_t allkeys;
xbps_dictionary_t idx = NULL, idxmeta = NULL, idxfiles = NULL;
xbps_dictionary_t idx = NULL, idxmeta = NULL;
struct xbps_repo *repo;
struct cbdata cbd;
int rv = 0;
@ -135,11 +109,9 @@ index_clean(struct xbps_handle *xhp, const char *repodir)
_XBPS_RINDEX, strerror(errno));
return rv;
}
xbps_repo_open_idxfiles(repo);
idx = xbps_dictionary_copy(repo->idx);
idxmeta = xbps_dictionary_copy(repo->idxmeta);
idxfiles = xbps_dictionary_copy(repo->idxfiles);
if (idx == NULL || idxfiles == NULL) {
if (idx == NULL) {
fprintf(stderr, "%s: incomplete repository data file!\n", _XBPS_RINDEX);
rv = EINVAL;
goto out;
@ -159,41 +131,20 @@ index_clean(struct xbps_handle *xhp, const char *repodir)
char *keyname = NULL, *pkgname = NULL;
xbps_array_get_cstring(cbd.result, x, &keyname);
printf("index-files: removed entry %s\n", keyname);
printf("index: removed entry %s\n", keyname);
pkgname = xbps_pkg_name(keyname);
assert(pkgname);
xbps_dictionary_remove(idxfiles, keyname);
xbps_dictionary_remove(idx, pkgname);
free(pkgname);
free(keyname);
flush = true;
}
/*
* Second pass: find out obsolete entries on index-files.
*/
xbps_object_release(cbd.result);
xbps_object_release(allkeys);
cbd.idx = idx;
cbd.result = xbps_array_create();
allkeys = xbps_dictionary_all_keys(idxfiles);
(void)xbps_array_foreach_cb_multi(xhp, allkeys, idxfiles, idxfiles_cleaner_cb, &cbd);
for (unsigned int x = 0; x < xbps_array_count(cbd.result); x++) {
char *keyname = NULL;
xbps_array_get_cstring(cbd.result, x, &keyname);
printf("index-files: removed entry %s\n", keyname);
xbps_dictionary_remove(idxfiles, keyname);
free(keyname);
flush = true;
}
pthread_mutex_destroy(&cbd.mtx);
xbps_object_release(cbd.result);
xbps_object_release(allkeys);
if (flush) {
if (!repodata_flush(xhp, repodir, idx, idxfiles, idxmeta)) {
if (!repodata_flush(xhp, repodir, idx, idxmeta)) {
rv = errno;
fprintf(stderr, "failed to write repodata: %s\n",
strerror(errno));
@ -202,8 +153,6 @@ index_clean(struct xbps_handle *xhp, const char *repodir)
}
printf("index: %u packages registered.\n",
xbps_dictionary_count(idx));
printf("index-files: %u packages registered.\n",
xbps_dictionary_count(idxfiles));
out:
xbps_repo_close(repo, true);
@ -211,8 +160,6 @@ out:
xbps_object_release(idx);
if (idxmeta)
xbps_object_release(idxmeta);
if (idxfiles)
xbps_object_release(idxfiles);
return rv;
}

14
bin/xbps-rindex/repoflush.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2013 Juan Romero Pardines.
* Copyright (c) 2013-2014 Juan Romero Pardines.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -39,8 +39,7 @@
bool
repodata_flush(struct xbps_handle *xhp, const char *repodir,
xbps_dictionary_t idx, xbps_dictionary_t idxfiles,
xbps_dictionary_t meta)
xbps_dictionary_t idx, xbps_dictionary_t meta)
{
struct archive *ar;
char *repofile, *tname, *buf;
@ -82,15 +81,6 @@ repodata_flush(struct xbps_handle *xhp, const char *repodir,
if (rv != 0)
return false;
/* XBPS_REPOIDX_FILES */
buf = xbps_dictionary_externalize(idxfiles);
assert(buf);
rv = xbps_archive_append_buf(ar, buf, strlen(buf),
XBPS_REPOIDX_FILES, 0644, "root", "root");
free(buf);
if (rv != 0)
return false;
/* Write data to tempfile and rename */
archive_write_finish(ar);
fdatasync(repofd);

3
bin/xbps-rindex/sign.c
View File

@ -153,7 +153,6 @@ sign_repo(struct xbps_handle *xhp, const char *repodir,
rv = EINVAL;
goto out;
}
xbps_repo_open_idxfiles(repo);
/*
* If privkey not set, default to ~/.ssh/id_rsa.
*/
@ -286,7 +285,7 @@ sign_repo(struct xbps_handle *xhp, const char *repodir,
xbps_object_release(data);
data = NULL;
if (!repodata_flush(xhp, repodir, repo->idx, repo->idxfiles, meta)) {
if (!repodata_flush(xhp, repodir, repo->idx, meta)) {
fprintf(stderr, "failed to write repodata: %s\n", strerror(errno));
goto out;
}

40
include/xbps.h.in
View File

@ -48,7 +48,7 @@
*
* This header documents the full API for the XBPS Library.
*/
#define XBPS_API_VERSION "20141113"
#define XBPS_API_VERSION "20141113-1"
#ifndef XBPS_VERSION
#define XBPS_VERSION "UNSET"
@ -112,12 +112,6 @@
*/
#define XBPS_REPOIDX "index.plist"
/**
* @def XBPS_REPOIDX_FILES
* Filename for the repository index-files property list.
*/
#define XBPS_REPOIDX_FILES "index-files.plist"
/**
* @def XBPS_REPOIDX_META
* Filename for the repository index metadata property list.
@ -1188,12 +1182,6 @@ struct xbps_repo {
* Proplib dictionary associated with the repository index-meta.
*/
xbps_dictionary_t idxmeta;
/**
* @var idxfiles
*
* Proplib dictionary associated with the repository index-files.
*/
xbps_dictionary_t idxfiles;
/**
* @var uri
*
@ -1345,15 +1333,6 @@ struct xbps_repo *xbps_repo_open(struct xbps_handle *xhp, const char *url, bool
*/
void xbps_repo_close(struct xbps_repo *repo, bool lock);
/**
* Prepares the repository index-files.plist to have access to it.
* The repository must be opened previously with \a xbps_repo_open().
*
* @param[in] repo The repository object to use.
*/
void xbps_repo_open_idxfiles(struct xbps_repo *repo);
/**
*
* Returns a heap-allocated string with the repository local path.
@ -1668,6 +1647,23 @@ bool xbps_binpkg_exists(struct xbps_handle *xhp, xbps_dictionary_t pkgd);
*/
bool xbps_repository_is_remote(const char *uri);
/*
* Returns an allocated string with the full path to the binary package
* matching \a pkgd.
*
* The \a pkgd dictionary must contain the following objects:
* - architecture (string)
* - pkgver (string)
* - repository (string)
*
* @param[in] xhp The pointer to an xbps_handle struct.
* @param[in] pkgd The package dictionary to match.
*
* @return A malloc(3)ed buffer with the full path, NULL otherwise.
* The buffer should be free(3)d when it's no longer needed.
*/
char *xbps_repository_pkg_path(struct xbps_handle *xhp, xbps_dictionary_t pkgd);
/**
* Gets the name of a package string. Package strings are composed
* by a @<pkgname@>/@<version@> pair and separated by the <em>minus</em>

6
include/xbps_api_impl.h
View File

@ -127,12 +127,6 @@ bool HIDDEN xbps_remove_pkg_from_array_by_name(xbps_array_t, const char *);
bool HIDDEN xbps_remove_pkg_from_array_by_pattern(xbps_array_t, const char *);
bool HIDDEN xbps_remove_pkg_from_array_by_pkgver(xbps_array_t, const char *);
/**
* @private
* From lib/util.c
*/
char HIDDEN *xbps_repository_pkg_path(struct xbps_handle *, xbps_dictionary_t);
/**
* @private
* From lib/download.c

11
lib/repo.c
View File

@ -202,13 +202,6 @@ out:
return NULL;
}
void
xbps_repo_open_idxfiles(struct xbps_repo *repo)
{
assert(repo);
repo->idxfiles = repo_get_dict(repo);
}
void
xbps_repo_close(struct xbps_repo *repo, bool lock)
{
@ -225,10 +218,6 @@ xbps_repo_close(struct xbps_repo *repo, bool lock)
xbps_object_release(repo->idxmeta);
repo->idxmeta = NULL;
}
if (repo->idxfiles != NULL) {
xbps_object_release(repo->idxfiles);
repo->idxfiles = NULL;
}
if (lock && lockf(repo->fd, F_ULOCK, 0) == -1)
xbps_dbg_printf(repo->xhp, "[repo] failed to unlock %s: %s\n", repo->uri, strerror(errno));

2
lib/util.c
View File

@ -280,7 +280,7 @@ xbps_pkgpattern_version(const char *pkg)
return strpbrk(pkg, "><*?[]");
}
char HIDDEN *
char *
xbps_repository_pkg_path(struct xbps_handle *xhp, xbps_dictionary_t pkg_repod)
{
const char *pkgver, *arch, *repoloc;

60
tests/xbps/xbps-rindex/clean_test.sh
View File

@ -22,63 +22,7 @@ noremove_body() {
atf_check_equal ${result} 1
}
# 2nd test: make sure that entries are also removed from index-files.
atf_test_case filesclean
filesclean_head() {
atf_set "descr" "xbps-rindex(8) -c: index-files clean test"
}
filesclean_body() {
mkdir -p some_repo pkg_A
touch -f pkg_A/file00
cd some_repo
xbps-create -A noarch -n foo-1.0_1 -s "foo pkg" ../pkg_A
atf_check_equal $? 0
xbps-rindex -d -a $PWD/*.xbps
atf_check_equal $? 0
rm *.xbps
cd ..
xbps-rindex -c some_repo
atf_check_equal $? 0
result=$(xbps-query -r root -C empty.conf --repository=some_repo -o \*)
test -z "${result}"
atf_check_equal $? 0
}
# 3nd test: make sure that entries are removed from index-files on updates.
atf_test_case filesclean2
filesclean2_head() {
atf_set "descr" "xbps-rindex(8) -c: index-files clean test on updates"
}
filesclean2_body() {
mkdir -p some_repo pkg_A
touch -f pkg_A/file00
cd some_repo
xbps-create -A noarch -n foo-1.0_1 -s "foo pkg" ../pkg_A
atf_check_equal $? 0
xbps-rindex -d -a $PWD/*.xbps
atf_check_equal $? 0
xbps-create -A noarch -n foo-1.1_1 -s "foo pkg" ../pkg_A
atf_check_equal $? 0
xbps-rindex -d -a $PWD/*.xbps
atf_check_equal $? 0
cd ..
xbps-rindex -c some_repo
atf_check_equal $? 0
result="$(xbps-query -r root -C empty.conf --repository=some_repo -o \*)"
expected="foo-1.1_1: /file00 (some_repo)"
rv=0
if [ "$result" != "$expected" ]; then
rv=1
fi
atf_check_equal $rv 0
}
# 4th test: xbps issue #19.
# xbps issue #19.
# How to reproduce it:
# Generate pkg foo-1.0_1.
# Add it to the index of a local repo.
@ -109,7 +53,5 @@ issue19_body() {
atf_init_test_cases() {
atf_add_test_case noremove
atf_add_test_case filesclean
atf_add_test_case filesclean2
atf_add_test_case issue19
}