Added support to fetch remote repodata on the fly.

See the NEWS file for more information.

NEWS

@@ -1,5 +1,13 @@
 xbps-0.42 (???):
 
+ * Added support to fetch repository data remotely and on demand. That means
+   that synchronizing the repository data with `xbps-install -S` is now
+   completely optional. If local repodata is found on disk, it's the preferred
+   operation mode. This also means that `xbps-query(8)` may be used by any
+   user without the need to be the `superuser` to synchronize the repository
+   archives. Of course this new operation mode requires a proper network
+   connection.
+
  * xbps-{install,query}: added new option (-i, --ignore-conf-repos) to ignore
    repositories defined in configuration files (xbps.d). Only repos specified
    by the command line will be used (--repository).

include/xbps.h.in

@@ -48,7 +48,7 @@
  *
  * This header documents the full API for the XBPS Library.
  */
-#define XBPS_API_VERSION	"20141121-1"
+#define XBPS_API_VERSION	"20141126"
 
 #ifndef XBPS_VERSION
  #define XBPS_VERSION		"UNSET"
@@ -1401,6 +1401,16 @@ void xbps_repo_close(struct xbps_repo *repo, bool lock);
  */
 char *xbps_repo_path(struct xbps_handle *xhp, const char *url);
 
+/**
+ * Remotely fetch repository data and keep it in memory.
+ *
+ * @param[in] repo A struct xbps_repo pointer to be filled in.
+ * @param[in] url Full url to the target remote repository data archive.
+ *
+ * @return True on success, false otherwise and errno is set appropiately.
+ */
+bool xbps_repo_fetch_remote(struct xbps_repo *repo, const char *url);
+
 /**
  * Returns a pkg dictionary from a repository \a repo matching
  * the expression \a pkg.

lib/plist_fetch.c

@@ -156,7 +156,9 @@ xbps_archive_fetch_file(const char *url, const char *fname)
 		const char *bfile;
 
 		bfile = archive_entry_pathname(entry);
-		bfile++; /* skip first dot */
+		if (bfile[0] == '.')
+			bfile++; /* skip first dot */
+
 		if (strcmp(bfile, fname) == 0) {
 			buf = xbps_archive_get_file(a, entry);
 			break;
@@ -168,6 +170,54 @@ xbps_archive_fetch_file(const char *url, const char *fname)
 	return buf;
 }
 
+bool
+xbps_repo_fetch_remote(struct xbps_repo *repo, const char *url)
+{
+	struct archive *a;
+	struct archive_entry *entry;
+	uint8_t i = 0;
+
+	assert(url);
+	assert(repo);
+
+	if ((a = open_archive(url)) == NULL)
+		return false;
+
+	while ((archive_read_next_header(a, &entry)) == ARCHIVE_OK) {
+		const char *bfile;
+		char *buf;
+
+		bfile = archive_entry_pathname(entry);
+		if (bfile[0] == '.')
+			bfile++; /* skip first dot */
+
+		if (strcmp(bfile, "index-meta.plist") == 0) {
+			buf = xbps_archive_get_file(a, entry);
+			repo->idxmeta = xbps_dictionary_internalize(buf);
+			free(buf);
+			i++;
+		} else if (strcmp(bfile, "index.plist") == 0) {
+			buf = xbps_archive_get_file(a, entry);
+			repo->idx = xbps_dictionary_internalize(buf);
+			free(buf);
+			i++;
+		} else {
+			archive_read_data_skip(a);
+		}
+		if (i == 2)
+			break;
+	}
+	archive_read_finish(a);
+
+	if (xbps_object_type(repo->idxmeta) == XBPS_TYPE_DICTIONARY)
+		repo->is_signed = true;
+
+	if (xbps_object_type(repo->idx) == XBPS_TYPE_DICTIONARY)
+		return true;
+
+	return false;
+}
+
 int
 xbps_archive_fetch_file_into_fd(const char *url, const char *fname, int fd)
 {
@@ -186,7 +236,9 @@ xbps_archive_fetch_file_into_fd(const char *url, const char *fname, int fd)
 		const char *bfile;
 
 		bfile = archive_entry_pathname(entry);
-		bfile++; /* skip first dot */
+		if (bfile[0] == '.')
+			bfile++; /* skip first dot */
+
 		if (strcmp(bfile, fname) == 0) {
 			rv = archive_read_data_into_fd(a, fd);
 			if (rv != 0)

lib/repo.c

@@ -106,11 +106,74 @@ repo_get_dict(struct xbps_repo *repo)
 	return d;
 }
 
+static bool
+repo_open_local(struct xbps_repo *repo, bool lock)
+{
+	struct stat st;
+	int rv = 0;
+
+	/*
+	 * Acquire a POSIX file lock on the archive; wait if the lock is
+	 * already taken.
+	 */
+        if (lock && lockf(repo->fd, F_LOCK, 0) == -1) {
+		rv = errno;
+		xbps_dbg_printf(repo->xhp, "[repo] failed to lock %s: %s\n", repo->uri, strerror(rv));
+		return false;
+	}
+	if (fstat(repo->fd, &st) == -1) {
+		rv = errno;
+		xbps_dbg_printf(repo->xhp, "[repo] `%s' fstat repodata %s\n",
+		    repo->uri, strerror(rv));
+		return false;
+	}
+
+	repo->ar = archive_read_new();
+	archive_read_support_compression_gzip(repo->ar);
+	archive_read_support_format_tar(repo->ar);
+
+	if (archive_read_open_fd(repo->ar, repo->fd, st.st_blksize) == ARCHIVE_FATAL) {
+		rv = archive_errno(repo->ar);
+		xbps_dbg_printf(repo->xhp,
+		    "[repo] `%s' failed to open repodata archive %s\n",
+		    repo->uri, strerror(rv));
+		return false;
+	}
+	if ((repo->idx = repo_get_dict(repo)) == NULL) {
+		rv = archive_errno(repo->ar);
+		xbps_dbg_printf(repo->xhp,
+		    "[repo] `%s' failed to internalize index on archive: %s\n",
+		    repo->uri, strerror(rv));
+		return false;
+	}
+	repo->idxmeta = repo_get_dict(repo);
+	if (repo->idxmeta != NULL)
+		repo->is_signed = true;
+
+	return true;
+}
+
+static bool
+repo_open_remote(struct xbps_repo *repo)
+{
+	char *rpath;
+	bool rv;
+
+	rpath = xbps_repo_path(repo->xhp, repo->uri);
+	rv = xbps_repo_fetch_remote(repo, rpath);
+	free(rpath);
+	if (rv) {
+		xbps_dbg_printf(repo->xhp, "[repo] `%s' used remotely (kept in memory).\n", repo->uri);
+		if (repo->xhp->state_cb && xbps_repo_key_import(repo) != 0)
+			rv = false;
+	}
+	return rv;
+}
+
 struct xbps_repo *
 xbps_repo_open(struct xbps_handle *xhp, const char *url, bool lock)
 {
 	struct xbps_repo *repo;
-	struct stat st;
 	const char *arch;
 	char *repofile;
 
@@ -151,46 +214,16 @@ xbps_repo_open(struct xbps_handle *xhp, const char *url, bool lock)
 		repo->fd = open(repofile, O_RDONLY);
 
 	if (repo->fd == -1) {
+		int rv = errno;
+		if (repo_open_remote(repo))
+			return repo;
+
 		xbps_dbg_printf(xhp, "[repo] `%s' open repodata %s\n",
-		    repofile, strerror(errno));
+		    repofile, strerror(rv));
 		goto out;
 	}
-	/*
+	if (repo_open_local(repo, lock))
-	 * Acquire a POSIX file lock on the archive; wait if the lock is
+		return repo;
-	 * already taken.
-	 */
-        if (lock && lockf(repo->fd, F_LOCK, 0) == -1) {
-		xbps_dbg_printf(xhp, "[repo] failed to lock %s: %s\n", repo->uri, strerror(errno));
-		goto out;
-	}
-	if (fstat(repo->fd, &st) == -1) {
-		xbps_dbg_printf(xhp, "[repo] `%s' fstat repodata %s\n",
-		    repofile, strerror(errno));
-		goto out;
-	}
-
-	repo->ar = archive_read_new();
-	archive_read_support_compression_gzip(repo->ar);
-	archive_read_support_format_tar(repo->ar);
-
-	if (archive_read_open_fd(repo->ar, repo->fd, st.st_blksize) == ARCHIVE_FATAL) {
-		xbps_dbg_printf(xhp,
-		    "[repo] `%s' failed to open repodata archive %s\n",
-		    repofile, strerror(archive_errno(repo->ar)));
-		goto out;
-	}
-	if ((repo->idx = repo_get_dict(repo)) == NULL) {
-		xbps_dbg_printf(xhp,
-		    "[repo] `%s' failed to internalize index on archive %s: %s\n",
-		    url, repofile, strerror(archive_errno(repo->ar)));
-		goto out;
-	}
-	repo->idxmeta = repo_get_dict(repo);
-	if (repo->idxmeta != NULL)
-		repo->is_signed = true;
-
-	free(repofile);
-	return repo;
 
 out:
 	if (repo->ar)