Improvements for xbps_fetch_file and xbps_repository_sync_index.

xbps_fetch_file:
	- A temp file is created with .part extension to improve resuming.
	- Files are downloaded in cwd.
	- Switch to futimens(2) and fsync(2).

xbps_repository_sync_index:
	- Do not create local repodir in metadir if it already exists.
	- Simplify the code thanks to new xbps_fetch_file().

bin/xbps-uhelper/main.c

@@ -265,7 +265,7 @@ main(int argc, char **argv)
 			usage();
 
 		for (i = 1; i < argc; i++) {
-			rv = xbps_fetch_file(&xh, argv[i], ".", false, "v");
+			rv = xbps_fetch_file(&xh, argv[i], "v");
 			if (rv == -1) {
 				printf("%s: %s\n", argv[1],
 				    xbps_fetch_error_string());

configure

@@ -205,7 +205,7 @@ fi
 
 case "$OS" in
 linux)
-	echo "CPPFLAGS += 	-D_XOPEN_SOURCE=500" >>$CONFIG_MK
+	echo "CPPFLAGS += 	-D_XOPEN_SOURCE=700" >>$CONFIG_MK
 	echo "CPPFLAGS += 	-D_FILE_OFFSET_BITS=64" >> $CONFIG_MK
 	;;
 *)

include/xbps_api.h.in

@@ -56,7 +56,7 @@
  */
 #define XBPS_PKGINDEX_VERSION	"1.5"
 
-#define XBPS_API_VERSION	"20121111"
+#define XBPS_API_VERSION	"20121111-1"
 
 #ifndef XBPS_VERSION
  #define XBPS_VERSION		"UNSET"
@@ -641,22 +641,16 @@ int xbps_configure_packages(struct xbps_handle *xhp, bool flush);
 
 
 /**
- * Download a file from a remote URL.
+ * Download a file from a remote URL to current working directory.
  * 
  * @param[in] xhp Pointer to an xbps_handle struct.
  * @param[in] uri Remote URI string.
- * @param[in] outputdir Directory string to store downloaded file.
- * @param[in] refetch If true and local/remote size/mtime do not match,
- * fetch the file from scratch.
  * @param[in] flags Flags passed to libfetch's fetchXget().
  * 
  * @return -1 on error, 0 if not downloaded (because local/remote size/mtime
  * do not match) and 1 if downloaded successfully.
  **/
-int xbps_fetch_file(struct xbps_handle *xhp,
+int xbps_fetch_file(struct xbps_handle *xhp, const char *uri,
-		    const char *uri,
-		    const char *outputdir,
-		    bool refetch,
 		    const char *flags);
 
 /**

lib/download.c

@@ -88,48 +88,53 @@ xbps_fetch_error_string(void)
 }
 
 int
-xbps_fetch_file(struct xbps_handle *xhp,
+xbps_fetch_file(struct xbps_handle *xhp, const char *uri, const char *flags)
-		const char *uri,
-		const char *outputdir,
-		bool refetch,
-		const char *flags)
 {
-	struct stat st;
+	struct stat st, st_tmpfile, *stp;
 	struct url *url = NULL;
 	struct url_stat url_st;
 	struct fetchIO *fio = NULL;
-	struct timeval tv[2];
+	struct timespec ts[2];
 	off_t bytes_dload = -1;
-	ssize_t bytes_read = -1, bytes_written;
+	ssize_t bytes_read = -1, bytes_written = -1;
-	char buf[4096], *filename, *destfile = NULL;
+	char buf[4096], *filename, *tempfile;
 	int fd = -1, rv = 0;
-	bool restart = false;
+	bool refetch = false, restart = false;
 
-	assert(uri != NULL);
+	assert(xhp);
-	assert(outputdir != NULL);
+	assert(uri);
 
+	/* Extern vars declared in libfetch */
 	fetchLastErrCode = 0;
-
 	fetchTimeout = xhp->fetch_timeout;
 	fetchRestartCalls = 1;
 	/*
 	 * Get the filename specified in URI argument.
 	 */
-	if ((filename = strrchr(uri, '/')) == NULL)
+	filename = strrchr(uri, '/') + 1;
+	if (filename == NULL)
 		return -1;
 
-	/* Skip first '/' */
+	tempfile = xbps_xasprintf("%s.part", filename);
-	filename++;
-	/*
-	 * Compute destination file path.
-	 */
-	destfile = xbps_xasprintf("%s/%s", outputdir, filename);
 	/*
 	 * Check if we have to resume a transfer.
 	 */
+	memset(&st_tmpfile, 0, sizeof(st_tmpfile));
+	if (stat(tempfile, &st_tmpfile) == 0) {
+		if (st_tmpfile.st_size > 0)
+			restart = true;
+	} else {
+		if (errno != ENOENT) {
+			rv = -1;
+			goto out;
+		}
+	}
+	/*
+	 * Check if we have to refetch a transfer.
+	 */
 	memset(&st, 0, sizeof(st));
-	if (stat(destfile, &st) == 0) {
+	if (stat(filename, &st) == 0) {
-		if (st.st_size > 0)
+		refetch = true;
 		restart = true;
 	} else {
 		if (errno != ENOENT) {
@@ -149,6 +154,7 @@ xbps_fetch_file(struct xbps_handle *xhp,
 	 * Check if we want to refetch from scratch a file.
 	 */
 	if (refetch) {
+		stp = &st;
 		/*
 		 * Issue a HEAD request to know size and mtime.
 		 */
@@ -159,8 +165,8 @@ xbps_fetch_file(struct xbps_handle *xhp,
 		 * If mtime and size match do nothing.
 		 */
 		if (restart && url_st.size && url_st.mtime &&
-		    url_st.size == st.st_size &&
+		    url_st.size == stp->st_size &&
-		    url_st.mtime == st.st_mtime)
+		    url_st.mtime == stp->st_mtime)
 			goto out;
 
 		/*
@@ -172,7 +178,7 @@ xbps_fetch_file(struct xbps_handle *xhp,
 		/*
 		 * Remove current file (if exists).
 		 */
-		if (restart && remove(destfile) == -1) {
+		if (restart && remove(tempfile) == -1) {
 			rv = -1;
 			goto out;
 		}
@@ -187,14 +193,15 @@ xbps_fetch_file(struct xbps_handle *xhp,
 		 * Issue a GET and skip the HEAD request, some servers
 		 * (googlecode.com) return a 404 in HEAD requests!
 		 */
-		url->offset = st.st_size;
+		stp = &st_tmpfile;
+		url->offset = stp->st_size;
 		fio = fetchXGet(url, &url_st, flags);
 	}
 
 	/* debug stuff */
-	xbps_dbg_printf(xhp, "st.st_size: %zd\n", (ssize_t)st.st_size);
+	xbps_dbg_printf(xhp, "st.st_size: %zd\n", (ssize_t)stp->st_size);
-	xbps_dbg_printf(xhp, "st.st_atime: %s\n", print_time(&st.st_atime));
+	xbps_dbg_printf(xhp, "st.st_atime: %s\n", print_time(&stp->st_atime));
-	xbps_dbg_printf(xhp, "st.st_mtime: %s\n", print_time(&st.st_mtime));
+	xbps_dbg_printf(xhp, "st.st_mtime: %s\n", print_time(&stp->st_mtime));
 	xbps_dbg_printf(xhp, "url->scheme: %s\n", url->scheme);
 	xbps_dbg_printf(xhp, "url->host: %s\n", url->host);
 	xbps_dbg_printf(xhp, "url->port: %d\n", url->port);
@@ -225,16 +232,18 @@ xbps_fetch_file(struct xbps_handle *xhp,
 		xbps_dbg_printf(xhp, "Remote file size is unknown, resume "
 		     "not possible...\n");
 		restart = false;
-	} else if (st.st_size > url_st.size) {
+	} else if (stp->st_size > url_st.size) {
 		/*
 		 * Remove local file if bigger than remote, and refetch the
 		 * whole shit again.
 		 */
 		xbps_dbg_printf(xhp, "Local file %s is greater than remote, "
 		    "removing local file and refetching...\n", filename);
-		(void)remove(destfile);
+		(void)remove(tempfile);
+		restart = false;
 	} else if (restart && url_st.mtime && url_st.size &&
-		   url_st.size == st.st_size && url_st.mtime == st.st_mtime) {
+		   url_st.size == stp->st_size &&
+		   url_st.mtime == stp->st_mtime) {
 		/* Local and remote size/mtime match, do nothing. */
 		goto out;
 	}
@@ -242,9 +251,9 @@ xbps_fetch_file(struct xbps_handle *xhp,
 	 * If restarting, open the file for appending otherwise create it.
 	 */
 	if (restart)
-		fd = open(destfile, O_WRONLY|O_APPEND);
+		fd = open(tempfile, O_WRONLY|O_APPEND);
 	else
-		fd = open(destfile, O_WRONLY|O_CREAT|O_TRUNC, 0644);
+		fd = open(tempfile, O_WRONLY|O_CREAT|O_TRUNC, 0644);
 
 	if (fd == -1) {
 		rv = -1;
@@ -263,7 +272,8 @@ xbps_fetch_file(struct xbps_handle *xhp,
 	while ((bytes_read = fetchIO_read(fio, buf, sizeof(buf))) > 0) {
 		bytes_written = write(fd, buf, (size_t)bytes_read);
 		if (bytes_written != bytes_read) {
-			xbps_dbg_printf(xhp, "Couldn't write to %s!\n", destfile);
+			xbps_dbg_printf(xhp,
+			    "Couldn't write to %s!\n", tempfile);
 			rv = -1;
 			goto out;
 		}
@@ -277,8 +287,8 @@ xbps_fetch_file(struct xbps_handle *xhp,
 		    filename, false, true, false);
 	}
 	if (bytes_read == -1) {
-		xbps_dbg_printf(xhp, "IO error while fetching %s: %s\n", filename,
+		xbps_dbg_printf(xhp, "IO error while fetching %s: %s\n",
-		    fetchLastErrString);
+		    filename, fetchLastErrString);
 		errno = EIO;
 		rv = -1;
 		goto out;
@@ -297,14 +307,24 @@ xbps_fetch_file(struct xbps_handle *xhp,
 	 * Update mtime in local file to match remote file if transfer
 	 * was successful.
 	 */
-	tv[0].tv_sec = url_st.atime ? url_st.atime : url_st.mtime;
+	ts[0].tv_sec = url_st.atime ? url_st.atime : url_st.mtime;
-	tv[1].tv_sec = url_st.mtime;
+	ts[1].tv_sec = url_st.mtime;
-	tv[0].tv_usec = tv[1].tv_usec = 0;
+	ts[0].tv_nsec = ts[1].tv_nsec = 0;
-	if (utimes(destfile, tv) == -1) {
+	if (futimens(fd, ts) == -1) {
+		rv = -1;
+		goto out;
+	}
+	/* sync and close fd */
+	(void)fsync(fd);
+	(void)close(fd);
+
+	/* File downloaded successfully, rename to destfile */
+	if (rename(tempfile, filename) == -1) {
+		xbps_dbg_printf(xhp, "failed to rename %s to %s: %s",
+		    tempfile, filename, strerror(errno));
 		rv = -1;
 		goto out;
 	}
-	/* File downloaded successfully */
 	rv = 1;
 
 out:
@@ -314,8 +334,8 @@ out:
 		fetchIO_close(fio);
 	if (url != NULL)
 		fetchFreeURL(url);
-	if (destfile != NULL)
+	if (tempfile != NULL)
-		free(destfile);
+		free(tempfile);
 
 	return rv;
 }

lib/external/fexec.c

@@ -27,6 +27,7 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
+#define _BSD_SOURCE	/* for vfork and chroot */
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <errno.h>

lib/repository_sync_index.c

@@ -23,7 +23,6 @@
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <sys/stat.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -88,70 +87,60 @@ xbps_repository_sync_pkg_index(struct xbps_handle *xhp,
 			       const char *plistf)
 {
 	prop_array_t array;
-	struct url *url = NULL;
+	const char *fetchstr = NULL;
-	struct stat st;
+	char *rpidx, *lrepodir, *uri_fixedp, *lrepofile;
-	const char *fetch_outputdir, *fetchstr = NULL;
-	char *rpidx, *lrepodir, *uri_fixedp;
-	char *tmp_metafile, *lrepofile;
 	int rv = 0;
-	bool only_sync = false;
 
 	assert(uri != NULL);
-	tmp_metafile = rpidx = lrepodir = lrepofile = NULL;
+	rpidx = uri_fixedp = lrepodir = lrepofile = NULL;
 
 	/* ignore non remote repositories */
 	if (!xbps_check_is_repository_uri_remote(uri))
 		return 0;
 
-	if ((url = fetchParseURL(uri)) == NULL)
-		return -1;
-
 	uri_fixedp = xbps_get_remote_repo_string(uri);
-	if (uri_fixedp == NULL) {
+	if (uri_fixedp == NULL)
-		fetchFreeURL(url);
 		return -1;
-	}
-	/*
-	 * Create metadir if necessary.
-	 */
-	if ((rv = xbps_mkpath(xhp->metadir, 0755)) == -1) {
-		xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL,
-		    errno, NULL, NULL,
-		    "[reposync] failed to create metadir `%s': %s",
-		    xhp->metadir, strerror(errno));
-		goto out;
-	}
 	/*
 	 * Remote repository plist index full URL.
 	 */
 	rpidx = xbps_xasprintf("%s/%s", uri, plistf);
-	/*
-	 * Save temporary file in metadir, and rename if it
-	 * was downloaded successfully.
-	 */
-	tmp_metafile = xbps_xasprintf("%s/%s", xhp->metadir, plistf);
 	/*
 	 * Full path to repository directory to store the plist
 	 * index file.
 	 */
 	lrepodir = xbps_xasprintf("%s/%s", xhp->metadir, uri_fixedp);
 	/*
-	 * If directory exists probably the plist index file
+	 * Full path to the local repository index file.
-	 * was downloaded previously...
 	 */
-	rv = stat(lrepodir, &st);
+	lrepofile = xbps_xasprintf("%s/%s", lrepodir, plistf);
-	if (rv == 0 && S_ISDIR(st.st_mode)) {
+	/*
-		only_sync = true;
+	 * Create repodir in metadir.
-		fetch_outputdir = lrepodir;
+	 */
-	} else
+	if (access(lrepodir, R_OK|X_OK|W_OK) == -1) {
-		fetch_outputdir = xhp->metadir;
+		if ((rv = xbps_mkpath(lrepodir, 0755)) == -1) {
+			xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL,
+			    errno, NULL, NULL,
+			    "[reposync] failed to create repodir `%s': %s",
+			    lrepodir, strerror(errno));
+			goto out;
+		}
+	}
+	if (chdir(lrepodir) == -1) {
+		xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL,
+		    errno, NULL, NULL,
+		    "[reposync] failed to change dir to repodir `%s': %s",
+		    lrepodir, strerror(errno));
+		rv = -1;
+		goto out;
+	}
 
 	/* reposync start cb */
 	xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC, 0, uri, plistf, NULL);
 	/*
 	 * Download plist index file from repository.
 	 */
-	if (xbps_fetch_file(xhp, rpidx, fetch_outputdir, true, NULL) == -1) {
+	if ((rv = xbps_fetch_file(xhp, rpidx, NULL)) == -1) {
 		/* reposync error cb */
 		fetchstr = xbps_fetch_error_string();
 		xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL,
@@ -159,59 +148,35 @@ xbps_repository_sync_pkg_index(struct xbps_handle *xhp,
 		    NULL, NULL,
 		    "[reposync] failed to fetch file `%s': %s",
 		    rpidx, fetchstr ? fetchstr : strerror(errno));
-		rv = -1;
 		goto out;
+	} else if (rv == 0) {
+	       goto out;
+	} else {
+		rv = 0;
 	}
-	if (only_sync)
-		goto out;
 	/*
 	 * Make sure that downloaded plist file can be internalized, i.e
 	 * some HTTP servers don't return proper errors and sometimes
-	  you get an HTML ASCII file :-)
+	 * you get an HTML ASCII file :-)
 	 */
-	array = prop_array_internalize_from_zfile(tmp_metafile);
+	array = prop_array_internalize_from_zfile(lrepofile);
 	if (array == NULL) {
 		xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL, 0, NULL, NULL,
 		    "[reposync] downloaded file `%s' is not valid.", rpidx);
-		(void)unlink(tmp_metafile);
+		(void)unlink(lrepofile);
+		(void)remove(lrepodir);
 		rv = -1;
 		goto out;
 	}
 	prop_object_release(array);
 
-	lrepofile = xbps_xasprintf("%s/%s", lrepodir, plistf);
-	/*
-	 * Create local repodir to store plist index file.
-	 */
-	if ((rv = xbps_mkpath(lrepodir, 0755)) == -1) {
-		xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL, errno, NULL, NULL,
-		    "[reposync] failed to create repodir for `%s': %s",
-		    lrepodir, strerror(rv));
-		goto out;
-	}
-
-	/*
-	 * Rename to destination file now it has been fetched successfully.
-	 */
-	if ((rv = rename(tmp_metafile, lrepofile)) == -1) {
-		xbps_set_cb_state(xhp, XBPS_STATE_REPOSYNC_FAIL, errno, NULL, NULL,
-		    "[reposync] failed to rename index file `%s' to `%s': %s",
-		    tmp_metafile, lrepofile, strerror(errno));
-	} else {
-		rv = 1; /* success */
-	}
-
 out:
 	if (rpidx)
 		free(rpidx);
 	if (lrepodir)
 		free(lrepodir);
-	if (tmp_metafile)
-		free(tmp_metafile);
 	if (lrepofile)
 		free(lrepofile);
-	if (url)
-		fetchFreeURL(url);
 	if (uri_fixedp)
 		free(uri_fixedp);
 

lib/transaction_commit.c

@@ -74,14 +74,10 @@ check_binpkgs_hash(struct xbps_handle *xhp, prop_object_iterator_t iter)
 		prop_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname);
 		prop_dictionary_get_cstring_nocopy(obj, "version", &version);
 		prop_dictionary_get_cstring_nocopy(obj, "repository", &repoloc);
-		assert(repoloc != NULL);
 		prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
-		assert(pkgver != NULL);
 		prop_dictionary_get_cstring_nocopy(obj, "filename", &filen);
-		assert(filen != NULL);
 		prop_dictionary_get_cstring_nocopy(obj,
 		    "filename-sha256", &sha256);
-		assert(sha256 != NULL);
 
 		binfile = xbps_path_from_repository_uri(xhp, obj, repoloc);
 		if (binfile == NULL) {
@@ -124,11 +120,8 @@ download_binpkgs(struct xbps_handle *xhp, prop_object_iterator_t iter)
 		prop_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname);
 		prop_dictionary_get_cstring_nocopy(obj, "version", &version);
 		prop_dictionary_get_cstring_nocopy(obj, "repository", &repoloc);
-		assert(repoloc != NULL);
 		prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);
-		assert(pkgver != NULL);
 		prop_dictionary_get_cstring_nocopy(obj, "filename", &filen);
-		assert(filen != NULL);
 
 		binfile = xbps_path_from_repository_uri(xhp, obj, repoloc);
 		if (binfile == NULL) {
@@ -145,15 +138,18 @@ download_binpkgs(struct xbps_handle *xhp, prop_object_iterator_t iter)
 		/*
 		 * Create cachedir.
 		 */
+		if (access(xhp->cachedir, R_OK|X_OK|W_OK) == -1) {
 			if (xbps_mkpath(xhp->cachedir, 0755) == -1) {
 				xbps_set_cb_state(xhp, XBPS_STATE_DOWNLOAD_FAIL,
 				    errno, pkgname, version,
-			    "%s: [trans] cannot create cachedir `%s': %s",
+				    "%s: [trans] cannot create cachedir `%s':"
-			    pkgver, xhp->cachedir, strerror(errno));
+				    "%s", pkgver, xhp->cachedir,
+				    strerror(errno));
 				free(binfile);
 				rv = errno;
 				break;
 			}
+		}
 		xbps_set_cb_state(xhp, XBPS_STATE_DOWNLOAD,
 		    0, pkgname, version,
 		    "Downloading binary package `%s' (from `%s')...",
@@ -161,7 +157,18 @@ download_binpkgs(struct xbps_handle *xhp, prop_object_iterator_t iter)
 		/*
 		 * Fetch binary package.
 		 */
-		rv = xbps_fetch_file(xhp, binfile, xhp->cachedir, false, NULL);
+		if (chdir(xhp->cachedir) == -1) {
+			xbps_set_cb_state(xhp, XBPS_STATE_DOWNLOAD_FAIL,
+			    errno, pkgname, version,
+			    "%s: [trans] failed to change dir to cachedir"
+			    "`%s': %s", pkgver, xhp->cachedir,
+			    strerror(errno));
+			rv = errno;
+			free(binfile);
+			break;
+		}
+
+		rv = xbps_fetch_file(xhp, binfile, NULL);
 		if (rv == -1) {
 			fetchstr = xbps_fetch_error_string();
 			xbps_set_cb_state(xhp, XBPS_STATE_DOWNLOAD_FAIL,