xbps-repo: changed the 'sync' target to update from all registered

remote repositories, so remove its argument.

xbps_sync_repository_pkg_index:
 - create the local repo dir in XBPS_META_PATH.
 - also add the uri scheme (http://, https://, ftp://) in the created
   local directory and subst ':' also with an underscore.
 - do not create local repo directories unless the download pkg-index
   plist file is verified to be fetched correctly.

xbps_check_is_repo_string_remote:
  New function to check if a repo string is a remote repository, use
  it in all places where it was used before.

--HG--
extra : convert_revision : xtraeme%40gmail.com-20091123042316-rmu4w3ehoxwh3iz8

bin/xbps-bin/install.c

@@ -122,8 +122,7 @@ download_package_list(prop_object_iterator_t iter)
 		/*
 		 * Skip packages in local repositories.
 		 */
-		if ((strncmp(repoloc, "http://", 7)) &&
+		if (!xbps_check_is_repo_string_remote(repoloc))
-		    (strncmp(repoloc, "ftp://", 6)))
 			continue;
 
 		prop_dictionary_get_cstring_nocopy(obj, "pkgver", &pkgver);

bin/xbps-repo/main.c

@@ -49,14 +49,13 @@ usage(void)
 {
 	printf("Usage: xbps-repo [options] [action] [arguments]\n\n"
 	" Available actions:\n"
-        "    add, genindex, list, remove, search, show\n"
+        "    add, genindex, list, remove, search, show, sync\n"
 	" Actions with arguments:\n"
 	"    add\t\t<URI>\n"
 	"    genindex\t<path>\n"
 	"    remove\t<URI>\n"
 	"    search\t<string>\n"
 	"    show\t<pkgname>\n"
-	"    sync\t\t<URI>\n"
 	" Options shared by all actions:\n"
 	"    -r\t\t<rootdir>\n"
 	"    -V\t\tPrints xbps release version\n"
@@ -134,21 +133,21 @@ out:
 }
 
 static int
-add_repository(const char *uri, bool remote)
+add_repository(const char *uri)
 {
 	prop_dictionary_t dict;
 	repo_info_t *rinfo;
 	char *plist, idxstr[PATH_MAX];
 	int rv = 0;
 
-	if (remote) {
+	if (xbps_check_is_repo_string_remote(uri)) {
 		if (!sanitize_localpath(idxstr, uri))
 			return errno;
 
 		printf("Fetching remote package index at %s...\n", uri);
 		rv = xbps_sync_repository_pkg_index(idxstr);
 		if (rv != 0) {
-			printf("Couldn't download pkg index: %s\n",
+			printf("Error: could not fetch pkg index file: %s.\n",
 			    xbps_fetch_error_string());
 			return rv;
 		}
@@ -208,8 +207,8 @@ int
 main(int argc, char **argv)
 {
 	char dpkgidx[PATH_MAX], *root = NULL;
+	struct repository_data *rdata = NULL;
 	int c, rv = 0;
-	bool remote_repo = false;
 
 	while ((c = getopt(argc, argv, "Vr:")) != -1) {
 		switch (c) {
@@ -238,11 +237,7 @@ main(int argc, char **argv)
 		if (argc != 2)
 			usage();
 
-		if ((strncmp(argv[1], "http://", 7) == 0) ||
+		rv = add_repository(argv[1]);
-		    (strncmp(argv[1], "ftp://", 6) == 0))
-			remote_repo = true;
-
-		rv = add_repository(argv[1], remote_repo);
 
 	} else if (strcasecmp(argv[0], "list") == 0) {
 		/* Lists all repositories registered in pool. */
@@ -304,15 +299,25 @@ main(int argc, char **argv)
 		exit(rv);
 
 	} else if (strcasecmp(argv[0], "sync") == 0) {
-		/* Syncs the pkg index file from a remote repo */
+		/* Syncs the pkg index for all registered remote repos */
-		if (argc != 2)
+		if (argc != 1)
 			usage();
 
-		if (!sanitize_localpath(dpkgidx, argv[1]))
+		if ((rv = xbps_prepare_repolist_data()) != 0)
-			exit(EXIT_FAILURE);
+			exit(rv);
-
+		/*
-		printf("Updating package index from: %s\n", dpkgidx);
+		 * Iterate over repository pool.
-		rv = xbps_sync_repository_pkg_index(dpkgidx);
+		 */
+		SIMPLEQ_FOREACH(rdata, &repodata_queue, chain) {
+			const char *uri = rdata->rd_uri;
+			if (xbps_check_is_repo_string_remote(uri)) {
+				printf("Syncing package index from: %s\n", uri);
+				rv = xbps_sync_repository_pkg_index(uri);
+				if (rv != 0)
+					break;
+			}
+		}
+		xbps_release_repolist_data();
 
 	} else {
 		usage();

bin/xbps-repo/xbps-repo.8.txt

@@ -67,10 +67,9 @@ Please note that all targets are *case insensitive*.
 	the size it takes in filesystem, description, maintainer, architecture
 	and other information.
 
-*sync 'URI'*::
+*sync*::
-	Syncs the package index file that is available from the repository
+	Syncs the package index file for all registered remote repositories.
-	specified at 'URI'. The new file will be fetched if it has been
+	The new file will be fetched if local and remote size/mtime do not match.
-	modified since last synchronization.
 
 
 BUGS

include/xbps_api.h

@@ -221,6 +221,7 @@ int SYMEXPORT		xbps_check_pkg_file_hash(prop_dictionary_t,
 						 const char *);
 int SYMEXPORT		xbps_check_is_installed_pkg(const char *);
 bool SYMEXPORT		xbps_check_is_installed_pkgname(const char *);
+bool SYMEXPORT		xbps_check_is_repo_string_remote(const char *);
 char SYMEXPORT		*xbps_get_pkg_index_plist(const char *);
 char SYMEXPORT		*xbps_get_pkg_name(const char *);
 char SYMEXPORT		*xbps_get_pkgdep_name(const char *);

lib/sync_remote_pkgidx.c

@@ -45,23 +45,23 @@ xbps_get_remote_repo_string(const char *uri)
 		return NULL;
 
 	/*
-	 * Replace dots and slashes with underscores, so that
+	 * Replace '.' ':' and '/' characters with underscores, so that
 	 * provided URL:
 	 *
-	 * 	www.foo.org/blah/xbps/binpkg-repo
+	 * 	http://www.foo.org/blah/xbps/binpkg-repo
 	 *
 	 * becomes:
 	 *
-	 * 	www_foo_org_blah_xbps_binpkg_repo
+	 * 	http___www_foo_org_blah_xbps_binpkg_repo
 	 * 	
 	 */
-	p = xbps_xasprintf("%s%s", url->host, url->doc);
+	p = xbps_xasprintf("%s://%s%s", url->scheme, url->host, url->doc);
 	fetchFreeURL(url);
 	if (p == NULL)
 		return NULL;
 
 	for (i = 0; i < strlen(p); i++) {
-		if (p[i] == '.' || p[i] == '/')
+		if (p[i] == '.' || p[i] == '/' || p[i] == ':')
 			p[i] = '_';
 	}
 
@@ -71,10 +71,17 @@ xbps_get_remote_repo_string(const char *uri)
 int SYMEXPORT
 xbps_sync_repository_pkg_index(const char *uri)
 {
-	struct url *url;
+	struct url *url = NULL;
 	struct utsname un;
-	char *rpidx, *dir, *lrepodir, *uri_fixedp = NULL;
+	struct stat st;
+	const char *fetch_outputdir;
+	char *rpidx, *dir, *lrepodir, *uri_fixedp;
+	char *metadir, *tmp_metafile, *lrepofile;
 	int rv = 0;
+	bool only_sync = false;
+
+	rpidx = dir = lrepodir = uri_fixedp = NULL;
+	metadir = tmp_metafile = lrepofile = NULL;
 
 	if (uname(&un) == -1)
 		return errno;
@@ -88,55 +95,137 @@ xbps_sync_repository_pkg_index(const char *uri)
 		return errno;
 	}
 
+	/*
+	 * Create metadir if it doesn't exist yet.
+	 */
+	metadir = xbps_xasprintf("%s/%s", xbps_get_rootdir(),
+	    XBPS_META_PATH);
+	if (metadir == NULL) {
+		rv = errno;
+		goto out;
+	}
+	rv = stat(metadir, &st);
+	if (rv == -1 && errno == ENOENT) {
+		if (mkpath(metadir, 0755) == -1) {
+			rv = errno;
+			goto out;
+		}
+	} else if (rv == 0 && !S_ISDIR(st.st_mode)) {
+		rv = ENOTDIR;
+		goto out;
+	}
+
+	/*
+	 * Remote repository pkg-index.plist full URL.
+	 */
+	rpidx = xbps_xasprintf("%s/%s/%s", uri, un.machine, XBPS_PKGINDEX);
+	if (rpidx == NULL) {
+		rv = errno;
+		goto out;
+	}
+	/*
+	 * Save temporary file in XBPS_META_PATH, and rename if it
+	 * was downloaded successfully.
+	 */
+	tmp_metafile = xbps_xasprintf("%s/%s", metadir, XBPS_PKGINDEX);
+	if (tmp_metafile == NULL) {
+		rv = errno;
+		goto out;
+	}
+	/*
+	 * Full path to machine arch local repository directory.
+	 */
+	lrepodir = xbps_xasprintf("%s/%s/%s/%s",
+	    xbps_get_rootdir(), XBPS_META_PATH, uri_fixedp, un.machine);
+	if (lrepodir == NULL) {
+		rv = errno;
+		goto out;
+	}
+	/*
+	 * If directory exists probably the pkg-index.plist file
+	 * was downloaded previously...
+	 */
+	rv = stat(lrepodir, &st);
+	if (rv == 0 && S_ISDIR(st.st_mode)) {
+		only_sync = true;
+		fetch_outputdir = lrepodir;
+	} else
+		fetch_outputdir = metadir;
+
+	/*
+	 * Download pkg-index.plist file from repository.
+	 */
+	if ((rv = xbps_fetch_file(rpidx, fetch_outputdir,
+	     true, NULL)) != 0) {
+		(void)remove(tmp_metafile);
+		goto out;
+	}
+	if (only_sync)
+		goto out;
+
 	/*
 	 * Create local arch repodir:
 	 *
 	 * 	<rootdir>/var/db/xbps/repo/<url_path_blah>/<arch>
 	 */
-	lrepodir = xbps_xasprintf("%s/%s/repo/%s/%s",
+	rv = stat(lrepodir, &st);
-	    xbps_get_rootdir(), XBPS_META_PATH, uri_fixedp, un.machine);
+	if (rv == -1 && errno == ENOENT) {
-	if (lrepodir == NULL) {
+		if (mkpath(lrepodir, 0755) == -1) {
-		fetchFreeURL(url);
+			rv = errno;
-		free(uri_fixedp);
+			goto out;
-		return errno;
+		}
-	}
+	} else if (rv == 0 && !S_ISDIR(st.st_mode)) {
-	if (mkpath(lrepodir, 0755) == -1) {
+		rv = ENOTDIR;
-		free(lrepodir);
+		goto out;
-		free(uri_fixedp);
-		fetchFreeURL(url);
-		return errno;
 	}
 	/*
 	 * Create local noarch repodir:
 	 *
 	 * 	<rootdir>/var/db/xbps/repo/<url_path_blah>/noarch
 	 */
-	dir = xbps_xasprintf("%s/%s/repo/%s/noarch",
+	dir = xbps_xasprintf("%s/%s/%s/noarch",
 	    xbps_get_rootdir(), XBPS_META_PATH, uri_fixedp);
-	free(uri_fixedp);
-	fetchFreeURL(url);
 	if (dir == NULL) {
-		free(lrepodir);
+		rv = errno;
-		return errno;
+		goto out;
 	}
-	if (mkpath(dir, 0755) == -1) {
+	rv = stat(dir, &st);
+	if (rv == -1 && errno == ENOENT) {
+		if (mkpath(dir, 0755) == -1) {
+			free(dir);
+			rv = errno;
+			goto out;
+		}
+	} else if (rv == 0 && !S_ISDIR(st.st_mode)) {
 		free(dir);
-		free(lrepodir);
+		rv = ENOTDIR;
-		return errno;
+		goto out;
 	}
 	free(dir);
-	/*
+	lrepofile = xbps_xasprintf("%s/%s", lrepodir, XBPS_PKGINDEX);
-	 * Download pkg-index.plist file from repository.
+	if (lrepofile == NULL) {
-	 */
+		rv = errno;
-	rpidx = xbps_xasprintf("%s/%s/%s", uri, un.machine, XBPS_PKGINDEX);
+		goto out;
-	if (rpidx == NULL) {
-		free(lrepodir);
-		return errno;
 	}
-	rv = xbps_fetch_file(rpidx, lrepodir, true, NULL);
+	/*
-
+	 * Rename to destination file now it has been fetched successfully.
-	free(rpidx);
+	 */
-	free(lrepodir);
+	rv = rename(tmp_metafile, lrepofile);
+out:
+	if (rpidx)
+		free(rpidx);
+	if (lrepodir)
+		free(lrepodir);
+	if (metadir)
+		free(metadir);
+	if (tmp_metafile)
+		free(tmp_metafile);
+	if (lrepofile)
+		free(lrepofile);
+	if (url)
+		fetchFreeURL(url);
+	if (uri_fixedp)
+		free(uri_fixedp);
 
 	return rv;
 }

lib/util.c

@@ -111,6 +111,19 @@ xbps_check_pkg_file_hash(prop_dictionary_t pkgd, const char *repoloc)
 	return rv;
 }
 
+bool SYMEXPORT
+xbps_check_is_repo_string_remote(const char *uri)
+{
+	assert(uri != NULL);
+
+	if ((strncmp(uri, "https://", 8) == 0) ||
+	    (strncmp(uri, "http://", 7) == 0) ||
+	    (strncmp(uri, "ftp://", 6) == 0))
+		return true;
+
+	return false;
+}
+
 int SYMEXPORT
 xbps_check_is_installed_pkg(const char *pkg)
 {
@@ -289,7 +302,7 @@ get_pkg_index_remote_plist(const char *uri, const char *machine)
 	if (uri_fixed == NULL)
 		return NULL;
 
-	repodir = xbps_xasprintf("%s/%s/repo/%s/%s/%s",
+	repodir = xbps_xasprintf("%s/%s/%s/%s/%s",
 	    xbps_get_rootdir(), XBPS_META_PATH, uri_fixed,
 	    machine, XBPS_PKGINDEX);
 	if (repodir == NULL) {