Implemented support for working with remote repositories.

libfetch from NetBSD's pkgsrc has been imported into lib/fetch, but
the objects are embedded into libxbps. Only a public function to fetch
files has been implemented: xbps_fetch_file().

The library now is built with -fvisibility=hidden by default, and
exported symbols are the ones that use the SYMEXPORT macro.

The code works well enough, but will need many more cleanups.

--HG--
extra : convert_revision : xtraeme%40gmail.com-20091027004600-0lq9aao67lisbzxv

bin/Makefile

@@ -3,6 +3,7 @@ include ../vars.mk
 SUBDIRS =	xbps-bin
 SUBDIRS +=	xbps-cmpver
 SUBDIRS +=	xbps-digest
+SUBDIRS +=	xbps-fetch
 SUBDIRS +=	xbps-pkgdb
 SUBDIRS +=	xbps-repo
 

bin/xbps-bin/install.c

@@ -108,6 +108,91 @@ check_pkg_hashes(prop_object_iterator_t iter)
 	return 0;
 }
 
+static int
+download_package_list(prop_object_iterator_t iter)
+{
+	prop_object_t obj;
+	const char *pkgname, *version, *repoloc, *filename, *arch;
+	char *savedir, *binfile, *lbinfile, *repoloc_trans;
+	int rv = 0;
+
+	printf("Downloading binary package file(s)...\n");
+	while ((obj = prop_object_iterator_next(iter)) != NULL) {
+		prop_dictionary_get_cstring_nocopy(obj, "repository", &repoloc);
+		/*
+		 * Skip packages in local repositories.
+		 */
+		if ((strncmp(repoloc, "http://", 7)) &&
+		    (strncmp(repoloc, "ftp://", 6)))
+			continue;
+
+		prop_dictionary_get_cstring_nocopy(obj, "pkgname", &pkgname);
+		prop_dictionary_get_cstring_nocopy(obj, "version", &version);
+		prop_dictionary_get_cstring_nocopy(obj, "filename", &filename);
+		prop_dictionary_get_cstring_nocopy(obj, "architecture", &arch);
+
+		repoloc_trans = xbps_get_remote_repo_string(repoloc);
+		if (repoloc_trans == NULL)
+			return errno;
+
+		savedir = xbps_xasprintf("%s/%s/repo/%s/%s",
+		    xbps_get_rootdir(), XBPS_META_PATH, repoloc_trans, arch);
+		if (savedir == NULL) {
+			free(repoloc_trans);
+			return errno;
+		}
+
+		lbinfile = xbps_xasprintf("%s/%s", savedir, filename);
+		if (lbinfile == NULL) {
+			free(repoloc_trans);
+			free(savedir);
+			return errno;
+		}
+
+		if (access(lbinfile, R_OK) == 0) {
+			free(savedir);
+			free(lbinfile);
+			goto change_repodir;
+		}
+		free(lbinfile);
+
+		binfile = xbps_xasprintf("%s/%s/%s", repoloc, arch, filename);
+		if (binfile == NULL) {
+			free(repoloc_trans);
+			free(savedir);
+			return errno;
+		}
+		printf("Downloading %s-%s binary package ...\n",
+		    pkgname, version);
+		rv = xbps_fetch_file(binfile, savedir);
+		free(savedir);
+		free(binfile);
+		if (rv != 0) {
+			printf("Couldn't download %s from %s (%s)\n",
+			    filename, repoloc, strerror(rv));
+			free(repoloc_trans);
+			return errno;
+		}
+
+change_repodir:
+		/*
+		 * If it was downloaded successfully, override repository
+		 * path in transaction dictionary.
+		 */
+		savedir = xbps_xasprintf("%s/%s/repo/%s",
+		    xbps_get_rootdir(), XBPS_META_PATH, repoloc_trans);
+		free(repoloc_trans);
+		if (savedir == NULL)
+		       return errno;
+
+		prop_dictionary_set_cstring(obj, "repository", savedir);
+		free(savedir);
+	}
+	prop_object_iterator_reset(iter);
+
+	return 0;
+}
+
 static void
 show_package_list(prop_object_iterator_t iter, const char *match)
 {
@@ -409,6 +494,12 @@ exec_transaction(struct transaction *trans)
 		}
 	}
 
+	/*
+	 * Download binary packages if they are in a remote repository.
+	 */
+	if ((rv = download_package_list(trans->iter)) != 0)
+		return rv;
+
 	/*
 	 * Check the SHA256 hash for all required packages.
 	 */

bin/xbps-digest/main.c

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2008 Juan Romero Pardines.
+ * Copyright (c) 2008-2009 Juan Romero Pardines.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -30,7 +30,6 @@
 #include <string.h>
 
 #include <xbps_api.h>
-#include "sha256.h"
 
 static void
 usage(void)
@@ -42,35 +41,21 @@ usage(void)
 int
 main(int argc, char **argv)
 {
-	SHA256_CTX ctx;
-	uint8_t buffer[BUFSIZ * 20], *digest;
-	ssize_t bytes;
-	int i, fd;
+	char *hash;
+	int i;
 
 	if (argc < 2)
 		usage();
 
 	for (i = 1; i < argc; i++) {
-		if ((fd = open(argv[i], O_RDONLY)) == -1) {
-			printf("xbps-digest: cannot open %s (%s)\n", argv[i],
-			    strerror(errno));
+		hash = xbps_get_file_hash(argv[i]);
+		if (hash == NULL) {
+			printf("Couldn't get hash for %s (%s)\n",
+			    argv[i], strerror(errno));
 			exit(EXIT_FAILURE);
 		}
-
-		digest = malloc(SHA256_DIGEST_STRING_LENGTH);
-		if (digest == NULL) {
-			printf("xbps-digest: malloc failed (%s)\n",
-			    strerror(errno));
-			exit(EXIT_FAILURE);
-		}
-
-		SHA256_Init(&ctx);
-		while ((bytes = read(fd, buffer, sizeof(buffer))) > 0)
-			SHA256_Update(&ctx, buffer, (size_t)bytes);
-
-		printf("%s\n", SHA256_End(&ctx, digest));
-		free(digest);
-		close(fd);
+		printf("%s\n", hash);
+		free(hash);
 	}
 
 	exit(EXIT_SUCCESS);

bin/xbps-fetch/Makefile

@@ -0,0 +1,6 @@
+TOPDIR = ../..
+include $(TOPDIR)/vars.mk
+
+BIN = xbps-fetch
+
+include $(TOPDIR)/prog.mk

bin/xbps-fetch/main.c

@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <xbps_api.h>
+
+int
+main(int argc, char **argv)
+{
+	if (argc != 2) {
+		printf("Usage: xbps-fetch [options] URL\n");
+		exit(EXIT_FAILURE);
+	}
+
+	return xbps_fetch_file(argv[1], ".");
+}

bin/xbps-repo/index.c

@@ -63,8 +63,6 @@ repoidx_getdict(const char *pkgdir)
 
 		prop_dictionary_set(dict, "packages", array);
 		prop_object_release(array);
-		prop_dictionary_set_cstring_nocopy(dict,
-		    "location-local", pkgdir);
 		prop_dictionary_set_cstring_nocopy(dict,
 		    "pkgindex-version", XBPS_PKGINDEX_VERSION);
 	}

bin/xbps-repo/main.c

@@ -37,8 +37,6 @@
 
 typedef struct repository_info {
 	const char *index_version;
-	const char *location_local;
-	const char *location_remote;
 	uint64_t total_pkgs;
 } repo_info_t;
 
@@ -84,14 +82,6 @@ pkgindex_getinfo(prop_dictionary_t dict, repo_info_t *ri)
 	    "pkgindex-version", &ri->index_version))
 		return false;
 
-	if (!prop_dictionary_get_cstring_nocopy(dict,
-	    "location-local", &ri->location_local))
-		return false;
-
-	/* This one is optional, thus don't panic */
-	prop_dictionary_get_cstring_nocopy(dict, "location-remote",
-	    &ri->location_remote);
-
 	if (!prop_dictionary_get_uint64(dict, "total-pkgs",
 	    &ri->total_pkgs))
 		return false;
@@ -144,13 +134,79 @@ out:
 	return rv;
 }
 
+static int
+add_repository(const char *uri, bool remote)
+{
+	prop_dictionary_t dict;
+	repo_info_t *rinfo;
+	char *plist, idxstr[PATH_MAX];
+	int rv = 0;
+
+	if (remote) {
+		rv = xbps_sync_repository_pkg_index(uri);
+		if (rv != 0)
+			return rv;
+		plist = xbps_get_pkg_index_plist(uri);
+	} else {
+		if (!sanitize_localpath(idxstr, uri))
+			return errno;
+		plist = xbps_get_pkg_index_plist(idxstr);
+	}
+
+	if (plist == NULL)
+		return errno;
+
+	dict = prop_dictionary_internalize_from_file(plist);
+	if (dict == NULL) {
+		printf("Repository %s does not contain any "
+		    "xbps pkgindex file.\n", idxstr);
+		rv = errno;
+		goto out;
+	}
+
+	rinfo = malloc(sizeof(*rinfo));
+	if (rinfo == NULL) {
+		rv = errno;
+		goto out;
+	}
+
+	if (!pkgindex_getinfo(dict, rinfo)) {
+		printf("'%s' is incomplete.\n", plist);
+		rv = EINVAL;
+		goto out;
+	}
+
+	if (remote)
+		rv = xbps_register_repository(uri);
+	else
+		rv = xbps_register_repository(idxstr);
+
+	if (rv != 0) {
+		printf("ERROR: couldn't register repository (%s)\n",
+		    strerror(rv));
+		goto out;
+	}
+	
+	printf("Added repository at %s (%s) with %ju packages.\n",
+	       uri, rinfo->index_version, rinfo->total_pkgs);
+
+out:
+	if (dict != NULL)
+		prop_object_release(dict);
+	if (rinfo != NULL)
+		free(rinfo);
+	if (plist != NULL)
+		free(plist);
+
+	return rv;
+}
+
 int
 main(int argc, char **argv)
 {
-	prop_dictionary_t dict;
-	repo_info_t *rinfo = NULL;
-	char dpkgidx[PATH_MAX], *plist, *root = NULL;
+	char dpkgidx[PATH_MAX], *root = NULL;
 	int c, rv = 0;
+	bool remote_repo = false;
 
 	while ((c = getopt(argc, argv, "Vr:")) != -1) {
 		switch (c) {
@@ -179,53 +235,11 @@ main(int argc, char **argv)
 		if (argc != 2)
 			usage();
 
-		if (!sanitize_localpath(dpkgidx, argv[1]))
-			exit(EXIT_FAILURE);
+		if ((strncmp(argv[1], "http://", 7) == 0) ||
+		    (strncmp(argv[1], "ftp://", 6) == 0))
+			remote_repo = true;
 
-		/* Temp buffer to verify pkgindex file. */
-		plist = xbps_get_pkg_index_plist(dpkgidx);
-		if (plist == NULL)
-			exit(EXIT_FAILURE);
-
-		dict = prop_dictionary_internalize_from_file(plist);
-		if (dict == NULL) {
-			printf("Directory %s does not contain any "
-			    "xbps pkgindex file.\n", dpkgidx);
-			free(plist);
-			exit(EXIT_FAILURE);
-		}
-
-		rinfo = malloc(sizeof(*rinfo));
-		if (rinfo == NULL) {
-			prop_object_release(dict);
-			free(plist);
-			exit(EXIT_FAILURE);
-		}
-
-		if (!pkgindex_getinfo(dict, rinfo)) {
-			printf("'%s' is incomplete.\n", plist);
-			prop_object_release(dict);
-			free(rinfo);
-			free(plist);
-			exit(EXIT_FAILURE);
-		}
-
-		if ((rv = xbps_register_repository(dpkgidx)) != 0) {
-			printf("ERROR: couldn't register repository (%s)\n",
-			    strerror(rv));
-			prop_object_release(dict);
-			free(rinfo);
-			free(plist);
-			exit(EXIT_FAILURE);
-		}
-
-		printf("Added repository at %s (%s) with %ju packages.\n",
-		       rinfo->location_local, rinfo->index_version,
-		       rinfo->total_pkgs);
-
-		prop_object_release(dict);
-		free(rinfo);
-		free(plist);
+		rv = add_repository(argv[1], remote_repo);
 
 	} else if (strcasecmp(argv[0], "list") == 0) {
 		/* Lists all repositories registered in pool. */

bin/xbps-repo/util.c

@@ -248,10 +248,8 @@ int
 show_pkg_info_from_repolist(prop_object_t obj, void *arg, bool *loop_done)
 {
 	prop_dictionary_t dict, pkgdict;
-	prop_string_t oloc;
-	const char *repofile, *repoloc;
+	const char *repofile;
 	char *plist;
-
 	assert(prop_object_type(obj) == PROP_TYPE_STRING);
 
 	/* Get the location */
@@ -276,19 +274,7 @@ show_pkg_info_from_repolist(prop_object_t obj, void *arg, bool *loop_done)
 		return 0;
 	}
 
-	oloc = prop_dictionary_get(dict, "location-remote");
-	if (oloc == NULL)
-		oloc = prop_dictionary_get(dict, "location-local");
-
-	if (oloc && prop_object_type(oloc) == PROP_TYPE_STRING)
-		repoloc = prop_string_cstring_nocopy(oloc);
-	else {
-		prop_object_release(dict);
-		free(plist);
-		return EINVAL;
-	}
-
-	printf("Repository: %s\n", repoloc);
+	printf("Repository: %s\n", repofile);
 	show_pkg_info(pkgdict);
 	*loop_done = true;
 	prop_object_release(dict);