emo/xbps
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implemented support for working with remote repositories.

Browse Source 
libfetch from NetBSD's pkgsrc has been imported into lib/fetch, but
the objects are embedded into libxbps. Only a public function to fetch
files has been implemented: xbps_fetch_file().

The library now is built with -fvisibility=hidden by default, and
exported symbols are the ones that use the SYMEXPORT macro.

The code works well enough, but will need many more cleanups.

--HG--
extra : convert_revision : xtraeme%40gmail.com-20091027004600-0lq9aao67lisbzxv
This commit is contained in:
Juan RP
2009-10-27 01:46:00 +01:00
parent 6a855c0272
commit 7aebea684b
41 changed files with 6039 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
lib/fetch/Makefile Normal file
View File

@ -0,0 +1,25 @@
TOPDIR = ../..
include $(TOPDIR)/vars.mk
CFLAGS += -Wno-unused-macros -Wno-conversion
CPPFLAGS += -DFTP_COMBINE_CWDS -DNETBSD -I$(TOPDIR)/include
OBJS= fetch.o common.o ftp.o http.o file.o
INCS= common.h
GEN = ftperr.h httperr.h
.PHONY: all
all: $(OBJS)
%.o: %.c $(INCS) $(GEN)
$(CC) $(CPPFLAGS) $(CFLAGS) -c $<
ftperr.h: ftp.errors
./errlist.sh ftp_errlist FTP ftp.errors > $@
httperr.h: http.errors
./errlist.sh http_errlist HTTP http.errors > $@
.PHONY: clean
clean:
-rm -f $(GEN) $(OBJS)

985
lib/fetch/common.c Normal file
View File

@ -0,0 +1,985 @@
/* $NetBSD: common.c,v 1.21 2009/10/15 12:36:57 joerg Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav
* Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer
* in this position and unchanged.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD: common.c,v 1.53 2007/12/19 00:26:36 des Exp $
*/
#if HAVE_CONFIG_H
#include "config.h"
#endif
#ifndef NETBSD
#include <nbcompat.h>
#endif
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <sys/uio.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <ctype.h>
#include <errno.h>
#if defined(HAVE_INTTYPES_H) || defined(NETBSD)
#include <inttypes.h>
#endif
#ifndef NETBSD
#include <nbcompat/netdb.h>
#else
#include <netdb.h>
#endif
#include <pwd.h>
#include <stdarg.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include "fetch.h"
#include "common.h"
#define DECONST(x,y) ((x)(uintptr_t)(y))
/*** Local data **************************************************************/
/*
* Error messages for resolver errors
*/
static struct fetcherr netdb_errlist[] = {
#ifdef EAI_NODATA
{ EAI_NODATA, FETCH_RESOLV, "Host not found" },
#endif
{ EAI_AGAIN, FETCH_TEMP, "Transient resolver failure" },
{ EAI_FAIL, FETCH_RESOLV, "Non-recoverable resolver failure" },
{ EAI_NONAME, FETCH_RESOLV, "No address record" },
{ -1, FETCH_UNKNOWN, "Unknown resolver error" }
};
/* End-of-Line */
static const char ENDL[2] = "\r\n";
/*** Error-reporting functions ***********************************************/
/*
* Map error code to string
*/
static struct fetcherr *
fetch_finderr(struct fetcherr *p, int e)
{
while (p->num != -1 && p->num != e)
p++;
return (p);
}
/*
* Set error code
*/
void
fetch_seterr(struct fetcherr *p, int e)
{
p = fetch_finderr(p, e);
fetchLastErrCode = p->cat;
snprintf(fetchLastErrString, MAXERRSTRING, "%s", p->string);
}
/*
* Set error code according to errno
*/
void
fetch_syserr(void)
{
switch (errno) {
case 0:
fetchLastErrCode = FETCH_OK;
break;
case EPERM:
case EACCES:
case EROFS:
#ifdef EAUTH
case EAUTH:
#endif
#ifdef ENEEDAUTH
case ENEEDAUTH:
#endif
fetchLastErrCode = FETCH_AUTH;
break;
case ENOENT:
case EISDIR: /* XXX */
fetchLastErrCode = FETCH_UNAVAIL;
break;
case ENOMEM:
fetchLastErrCode = FETCH_MEMORY;
break;
case EBUSY:
case EAGAIN:
fetchLastErrCode = FETCH_TEMP;
break;
case EEXIST:
fetchLastErrCode = FETCH_EXISTS;
break;
case ENOSPC:
fetchLastErrCode = FETCH_FULL;
break;
case EADDRINUSE:
case EADDRNOTAVAIL:
case ENETDOWN:
case ENETUNREACH:
case ENETRESET:
case EHOSTUNREACH:
fetchLastErrCode = FETCH_NETWORK;
break;
case ECONNABORTED:
case ECONNRESET:
fetchLastErrCode = FETCH_ABORT;
break;
case ETIMEDOUT:
fetchLastErrCode = FETCH_TIMEOUT;
break;
case ECONNREFUSED:
case EHOSTDOWN:
fetchLastErrCode = FETCH_DOWN;
break;
default:
fetchLastErrCode = FETCH_UNKNOWN;
}
snprintf(fetchLastErrString, MAXERRSTRING, "%s", strerror(errno));
}
/*
* Emit status message
*/
void
fetch_info(const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
vfprintf(stderr, fmt, ap);
va_end(ap);
fputc('\n', stderr);
}
/*** Network-related utility functions ***************************************/
/*
* Return the default port for a scheme
*/
int
fetch_default_port(const char *scheme)
{
struct servent *se;
if ((se = getservbyname(scheme, "tcp")) != NULL)
return (ntohs(se->s_port));
if (strcasecmp(scheme, SCHEME_FTP) == 0)
return (FTP_DEFAULT_PORT);
if (strcasecmp(scheme, SCHEME_HTTP) == 0)
return (HTTP_DEFAULT_PORT);
return (0);
}
/*
* Return the default proxy port for a scheme
*/
int
fetch_default_proxy_port(const char *scheme)
{
if (strcasecmp(scheme, SCHEME_FTP) == 0)
return (FTP_DEFAULT_PROXY_PORT);
if (strcasecmp(scheme, SCHEME_HTTP) == 0)
return (HTTP_DEFAULT_PROXY_PORT);
return (0);
}
/*
* Create a connection for an existing descriptor.
*/
conn_t *
fetch_reopen(int sd)
{
conn_t *conn;
/* allocate and fill connection structure */
if ((conn = calloc(1, sizeof(*conn))) == NULL)
return (NULL);
conn->next_buf = NULL;
conn->next_len = 0;
conn->sd = sd;
conn->is_active = 0;
++conn->ref;
return (conn);
}
/*
* Bump a connection's reference count.
*/
conn_t *
fetch_ref(conn_t *conn)
{
++conn->ref;
return (conn);
}
/*
* Bind a socket to a specific local address
*/
int
fetch_bind(int sd, int af, const char *addr)
{
struct addrinfo hints, *res, *res0;
memset(&hints, 0, sizeof(hints));
hints.ai_family = af;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = 0;
if (getaddrinfo(addr, NULL, &hints, &res0))
return (-1);
for (res = res0; res; res = res->ai_next) {
if (bind(sd, res->ai_addr, res->ai_addrlen) == 0)
return (0);
}
return (-1);
}
/*
* Establish a TCP connection to the specified port on the specified host.
*/
conn_t *
fetch_connect(const char *host, int port, int af, int verbose)
{
conn_t *conn;
char pbuf[10];
const char *bindaddr;
struct addrinfo hints, *res, *res0;
int sd, error;
if (verbose)
fetch_info("looking up %s", host);
/* look up host name and set up socket address structure */
snprintf(pbuf, sizeof(pbuf), "%d", port);
memset(&hints, 0, sizeof(hints));
hints.ai_family = af;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = 0;
if ((error = getaddrinfo(host, pbuf, &hints, &res0)) != 0) {
netdb_seterr(error);
return (NULL);
}
bindaddr = getenv("FETCH_BIND_ADDRESS");
if (verbose)
fetch_info("connecting to %s:%d", host, port);
/* try to connect */
for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) {
if ((sd = socket(res->ai_family, res->ai_socktype,
res->ai_protocol)) == -1)
continue;
if (bindaddr != NULL && *bindaddr != '\0' &&
fetch_bind(sd, res->ai_family, bindaddr) != 0) {
fetch_info("failed to bind to '%s'", bindaddr);
close(sd);
continue;
}
if (connect(sd, res->ai_addr, res->ai_addrlen) == 0)
break;
close(sd);
}
freeaddrinfo(res0);
if (sd == -1) {
fetch_syserr();
return (NULL);
}
if ((conn = fetch_reopen(sd)) == NULL) {
fetch_syserr();
close(sd);
}
return (conn);
}
/*
* Enable SSL on a connection.
*/
int
fetch_ssl(conn_t *conn, int verbose)
{
#ifdef WITH_SSL
/* Init the SSL library and context */
if (!SSL_library_init()){
fprintf(stderr, "SSL library init failed\n");
return (-1);
}
SSL_load_error_strings();
conn->ssl_meth = SSLv23_client_method();
conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth);
SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY);
conn->ssl = SSL_new(conn->ssl_ctx);
if (conn->ssl == NULL){
fprintf(stderr, "SSL context creation failed\n");
return (-1);
}
SSL_set_fd(conn->ssl, conn->sd);
if (SSL_connect(conn->ssl) == -1){
ERR_print_errors_fp(stderr);
return (-1);
}
if (verbose) {
X509_NAME *name;
char *str;
fprintf(stderr, "SSL connection established using %s\n",
SSL_get_cipher(conn->ssl));
conn->ssl_cert = SSL_get_peer_certificate(conn->ssl);
name = X509_get_subject_name(conn->ssl_cert);
str = X509_NAME_oneline(name, 0, 0);
printf("Certificate subject: %s\n", str);
free(str);
name = X509_get_issuer_name(conn->ssl_cert);
str = X509_NAME_oneline(name, 0, 0);
printf("Certificate issuer: %s\n", str);
free(str);
}
return (0);
#else
(void)conn;
(void)verbose;
fprintf(stderr, "SSL support disabled\n");
return (-1);
#endif
}
/*
* Read a character from a connection w/ timeout
*/
ssize_t
fetch_read(conn_t *conn, char *buf, size_t len)
{
struct timeval now, timeout, waittv;
fd_set readfds;
ssize_t rlen;
int r;
if (len == 0)
return 0;
if (conn->next_len != 0) {
if (conn->next_len < len)
len = conn->next_len;
memmove(buf, conn->next_buf, len);
conn->next_len -= len;
conn->next_buf += len;
return len;
}
if (fetchTimeout) {
FD_ZERO(&readfds);
gettimeofday(&timeout, NULL);
timeout.tv_sec += fetchTimeout;
}
for (;;) {
while (fetchTimeout && !FD_ISSET(conn->sd, &readfds)) {
FD_SET(conn->sd, &readfds);
gettimeofday(&now, NULL);
waittv.tv_sec = timeout.tv_sec - now.tv_sec;
waittv.tv_usec = timeout.tv_usec - now.tv_usec;
if (waittv.tv_usec < 0) {
waittv.tv_usec += 1000000;
waittv.tv_sec--;
}
if (waittv.tv_sec < 0) {
errno = ETIMEDOUT;
fetch_syserr();
return (-1);
}
errno = 0;
r = select(conn->sd + 1, &readfds, NULL, NULL, &waittv);
if (r == -1) {
if (errno == EINTR && fetchRestartCalls)
continue;
fetch_syserr();
return (-1);
}
}
#ifdef WITH_SSL
if (conn->ssl != NULL)
rlen = SSL_read(conn->ssl, buf, len);
else
#endif
rlen = read(conn->sd, buf, len);
if (rlen >= 0)
break;
if (errno != EINTR || !fetchRestartCalls)
return (-1);
}
return (rlen);
}
/*
* Read a line of text from a connection w/ timeout
*/
#define MIN_BUF_SIZE 1024
int
fetch_getln(conn_t *conn)
{
char *tmp, *next;
size_t tmpsize;
ssize_t len;
if (conn->buf == NULL) {
if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL) {
errno = ENOMEM;
return (-1);
}
conn->bufsize = MIN_BUF_SIZE;
}
conn->buflen = 0;
next = NULL;
do {
/*
* conn->bufsize != conn->buflen at this point,
* so the buffer can be NUL-terminated below for
* the case of len == 0.
*/
len = fetch_read(conn, conn->buf + conn->buflen,
conn->bufsize - conn->buflen);
if (len == -1)
return (-1);
if (len == 0)
break;
next = memchr(conn->buf + conn->buflen, '\n', len);
conn->buflen += len;
if (conn->buflen == conn->bufsize && next == NULL) {
tmp = conn->buf;
tmpsize = conn->bufsize * 2;
if (tmpsize < conn->bufsize) {
errno = ENOMEM;
return (-1);
}
if ((tmp = realloc(tmp, tmpsize)) == NULL) {
errno = ENOMEM;
return (-1);
}
conn->buf = tmp;
conn->bufsize = tmpsize;
}
} while (next == NULL);
if (next != NULL) {
*next = '\0';
conn->next_buf = next + 1;
conn->next_len = conn->buflen - (conn->next_buf - conn->buf);
conn->buflen = next - conn->buf;
} else {
conn->buf[conn->buflen] = '\0';
conn->next_len = 0;
}
return (0);
}
/*
* Write to a connection w/ timeout
*/
ssize_t
fetch_write(conn_t *conn, const char *buf, size_t len)
{
struct iovec iov;
iov.iov_base = DECONST(char *, buf);
iov.iov_len = len;
return fetch_writev(conn, &iov, 1);
}
/*
* Write a vector to a connection w/ timeout
* Note: can modify the iovec.
*/
ssize_t
fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
{
struct timeval now, timeout, waittv;
fd_set writefds;
ssize_t wlen, total;
int r;
if (fetchTimeout) {
FD_ZERO(&writefds);
gettimeofday(&timeout, NULL);
timeout.tv_sec += fetchTimeout;
}
total = 0;
while (iovcnt > 0) {
while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) {
FD_SET(conn->sd, &writefds);
gettimeofday(&now, NULL);
waittv.tv_sec = timeout.tv_sec - now.tv_sec;
waittv.tv_usec = timeout.tv_usec - now.tv_usec;
if (waittv.tv_usec < 0) {
waittv.tv_usec += 1000000;
waittv.tv_sec--;
}
if (waittv.tv_sec < 0) {
errno = ETIMEDOUT;
fetch_syserr();
return (-1);
}
errno = 0;
r = select(conn->sd + 1, NULL, &writefds, NULL, &waittv);
if (r == -1) {
if (errno == EINTR && fetchRestartCalls)
continue;
return (-1);
}
}
errno = 0;
#ifdef WITH_SSL
if (conn->ssl != NULL)
wlen = SSL_write(conn->ssl,
iov->iov_base, iov->iov_len);
else
#endif
wlen = writev(conn->sd, iov, iovcnt);
if (wlen == 0) {
/* we consider a short write a failure */
errno = EPIPE;
fetch_syserr();
return (-1);
}
if (wlen < 0) {
if (errno == EINTR && fetchRestartCalls)
continue;
return (-1);
}
total += wlen;
while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) {
wlen -= iov->iov_len;
iov++;
iovcnt--;
}
if (iovcnt > 0) {
iov->iov_len -= wlen;
iov->iov_base = DECONST(char *, iov->iov_base) + wlen;
}
}
return (total);
}
/*
* Write a line of text to a connection w/ timeout
*/
int
fetch_putln(conn_t *conn, const char *str, size_t len)
{
struct iovec iov[2];
ssize_t ret;
iov[0].iov_base = DECONST(char *, str);
iov[0].iov_len = len;
iov[1].iov_base = DECONST(char *, ENDL);
iov[1].iov_len = sizeof(ENDL);
if (len == 0)
ret = fetch_writev(conn, &iov[1], 1);
else
ret = fetch_writev(conn, iov, 2);
if (ret == -1)
return (-1);
return (0);
}
/*
* Close connection
*/
int
fetch_close(conn_t *conn)
{
int ret;
if (--conn->ref > 0)
return (0);
ret = close(conn->sd);
free(conn->buf);
free(conn);
return (ret);
}
/*** Directory-related utility functions *************************************/
int
fetch_add_entry(struct url_list *ue, struct url *base, const char *name,
int pre_quoted)
{
struct url *tmp;
char *tmp_name;
size_t base_doc_len, name_len, i;
unsigned char c;
if (strchr(name, '/') != NULL ||
strcmp(name, "..") == 0 ||
strcmp(name, ".") == 0)
return 0;
if (strcmp(base->doc, "/") == 0)
base_doc_len = 0;
else
base_doc_len = strlen(base->doc);
name_len = 1;
for (i = 0; name[i] != '\0'; ++i) {
if ((!pre_quoted && name[i] == '%') ||
!fetch_urlpath_safe(name[i]))
name_len += 3;
else
++name_len;
}
tmp_name = malloc( base_doc_len + name_len + 1);
if (tmp_name == NULL) {
errno = ENOMEM;
fetch_syserr();
return (-1);
}
if (ue->length + 1 >= ue->alloc_size) {
tmp = realloc(ue->urls, (ue->alloc_size * 2 + 1) * sizeof(*tmp));
if (tmp == NULL) {
free(tmp_name);
errno = ENOMEM;
fetch_syserr();
return (-1);
}
ue->alloc_size = ue->alloc_size * 2 + 1;
ue->urls = tmp;
}
tmp = ue->urls + ue->length;
strcpy(tmp->scheme, base->scheme);
strcpy(tmp->user, base->user);
strcpy(tmp->pwd, base->pwd);
strcpy(tmp->host, base->host);
tmp->port = base->port;
tmp->doc = tmp_name;
memcpy(tmp->doc, base->doc, base_doc_len);
tmp->doc[base_doc_len] = '/';
for (i = base_doc_len + 1; *name != '\0'; ++name) {
if ((!pre_quoted && *name == '%') ||
!fetch_urlpath_safe(*name)) {
tmp->doc[i++] = '%';
c = (unsigned char)*name / 16;
if (c < 10)
tmp->doc[i++] = '0' + c;
else
tmp->doc[i++] = 'a' - 10 + c;
c = (unsigned char)*name % 16;
if (c < 10)
tmp->doc[i++] = '0' + c;
else
tmp->doc[i++] = 'a' - 10 + c;
} else {
tmp->doc[i++] = *name;
}
}
tmp->doc[i] = '\0';
tmp->offset = 0;
tmp->length = 0;
tmp->last_modified = -1;
++ue->length;
return (0);
}
void
fetchInitURLList(struct url_list *ue)
{
ue->length = ue->alloc_size = 0;
ue->urls = NULL;
}
int
fetchAppendURLList(struct url_list *dst, const struct url_list *src)
{
size_t i, j, len;
len = dst->length + src->length;
if (len > dst->alloc_size) {
struct url *tmp;
tmp = realloc(dst->urls, len * sizeof(*tmp));
if (tmp == NULL) {
errno = ENOMEM;
fetch_syserr();
return (-1);
}
dst->alloc_size = len;
dst->urls = tmp;
}
for (i = 0, j = dst->length; i < src->length; ++i, ++j) {
dst->urls[j] = src->urls[i];
dst->urls[j].doc = strdup(src->urls[i].doc);
if (dst->urls[j].doc == NULL) {
while (i-- > 0)
free(dst->urls[j].doc);
fetch_syserr();
return -1;
}
}
dst->length = len;
return 0;
}
void
fetchFreeURLList(struct url_list *ue)
{
size_t i;
for (i = 0; i < ue->length; ++i)
free(ue->urls[i].doc);
free(ue->urls);
ue->length = ue->alloc_size = 0;
}
/*** Authentication-related utility functions ********************************/
static const char *
fetch_read_word(FILE *f)
{
static char word[1024];
if (fscanf(f, " %1023s ", word) != 1)
return (NULL);
return (word);
}
/*
* Get authentication data for a URL from .netrc
*/
int
fetch_netrc_auth(struct url *url)
{
char fn[PATH_MAX];
const char *word;
char *p;
FILE *f;
if ((p = getenv("NETRC")) != NULL) {
if (snprintf(fn, sizeof(fn), "%s", p) >= (int)sizeof(fn)) {
fetch_info("$NETRC specifies a file name "
"longer than PATH_MAX");
return (-1);
}
} else {
if ((p = getenv("HOME")) != NULL) {
struct passwd *pwd;
if ((pwd = getpwuid(getuid())) == NULL ||
(p = pwd->pw_dir) == NULL)
return (-1);
}
if (snprintf(fn, sizeof(fn), "%s/.netrc", p) >= (int)sizeof(fn))
return (-1);
}
if ((f = fopen(fn, "r")) == NULL)
return (-1);
while ((word = fetch_read_word(f)) != NULL) {
if (strcmp(word, "default") == 0)
break;
if (strcmp(word, "machine") == 0 &&
(word = fetch_read_word(f)) != NULL &&
strcasecmp(word, url->host) == 0) {
break;
}
}
if (word == NULL)
goto ferr;
while ((word = fetch_read_word(f)) != NULL) {
if (strcmp(word, "login") == 0) {
if ((word = fetch_read_word(f)) == NULL)
goto ferr;
if (snprintf(url->user, sizeof(url->user),
"%s", word) > (int)sizeof(url->user)) {
fetch_info("login name in .netrc is too long");
url->user[0] = '\0';
}
} else if (strcmp(word, "password") == 0) {
if ((word = fetch_read_word(f)) == NULL)
goto ferr;
if (snprintf(url->pwd, sizeof(url->pwd),
"%s", word) > (int)sizeof(url->pwd)) {
fetch_info("password in .netrc is too long");
url->pwd[0] = '\0';
}
} else if (strcmp(word, "account") == 0) {
if ((word = fetch_read_word(f)) == NULL)
goto ferr;
/* XXX not supported! */
} else {
break;
}
}
fclose(f);
return (0);
ferr:
fclose(f);
return (-1);
}
/*
* The no_proxy environment variable specifies a set of domains for
* which the proxy should not be consulted; the contents is a comma-,
* or space-separated list of domain names. A single asterisk will
* override all proxy variables and no transactions will be proxied
* (for compatability with lynx and curl, see the discussion at
* <http://curl.haxx.se/mail/archive_pre_oct_99/0009.html>).
*/
int
fetch_no_proxy_match(const char *host)
{
const char *no_proxy, *p, *q;
size_t h_len, d_len;
if ((no_proxy = getenv("NO_PROXY")) == NULL &&
(no_proxy = getenv("no_proxy")) == NULL)
return (0);
/* asterisk matches any hostname */
if (strcmp(no_proxy, "*") == 0)
return (1);
h_len = strlen(host);
p = no_proxy;
do {
/* position p at the beginning of a domain suffix */
while (*p == ',' || isspace((unsigned char)*p))
p++;
/* position q at the first separator character */
for (q = p; *q; ++q)
if (*q == ',' || isspace((unsigned char)*q))
break;
d_len = q - p;
if (d_len > 0 && h_len > d_len &&
strncasecmp(host + h_len - d_len,
p, d_len) == 0) {
/* domain name matches */
return (1);
}
p = q + 1;
} while (*q);
return (0);
}
struct fetchIO {
void *io_cookie;
ssize_t (*io_read)(void *, void *, size_t);
ssize_t (*io_write)(void *, const void *, size_t);
void (*io_close)(void *);
};
void
fetchIO_close(fetchIO *f)
{
if (f->io_close != NULL)
(*f->io_close)(f->io_cookie);
free(f);
}
fetchIO *
fetchIO_unopen(void *io_cookie, ssize_t (*io_read)(void *, void *, size_t),
ssize_t (*io_write)(void *, const void *, size_t),
void (*io_close)(void *))
{
fetchIO *f;
f = malloc(sizeof(*f));
if (f == NULL)
return f;
f->io_cookie = io_cookie;
f->io_read = io_read;
f->io_write = io_write;
f->io_close = io_close;
return f;
}
ssize_t
fetchIO_read(fetchIO *f, void *buf, size_t len)
{
if (f->io_read == NULL)
return EBADF;
return (*f->io_read)(f->io_cookie, buf, len);
}
ssize_t
fetchIO_write(fetchIO *f, const void *buf, size_t len)
{
if (f->io_read == NULL)
return EBADF;
return (*f->io_write)(f->io_cookie, buf, len);
}

137
lib/fetch/common.h Normal file
View File

@ -0,0 +1,137 @@
/* $NetBSD: common.h,v 1.12 2009/08/16 20:31:29 joerg Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer
* in this position and unchanged.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD: common.h,v 1.30 2007/12/18 11:03:07 des Exp $
*/
#ifndef _COMMON_H_INCLUDED
#define _COMMON_H_INCLUDED
#define FTP_DEFAULT_PORT 21
#define HTTP_DEFAULT_PORT 80
#define FTP_DEFAULT_PROXY_PORT 21
#define HTTP_DEFAULT_PROXY_PORT 3128
#ifdef WITH_SSL
#include <openssl/crypto.h>
#include <openssl/x509.h>
#include <openssl/pem.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#endif
#if !defined(__sun) && !defined(__hpux) && !defined(__INTERIX) && \
!defined(__digital__) && !defined(__linux) && !defined(__MINT__) && \
!defined(__sgi)
#define HAVE_SA_LEN
#endif
/* Connection */
typedef struct fetchconn conn_t;
struct fetchconn {
int sd; /* socket descriptor */
char *buf; /* buffer */
size_t bufsize; /* buffer size */
size_t buflen; /* length of buffer contents */
char *next_buf; /* pending buffer, e.g. after getln */
size_t next_len; /* size of pending buffer */
int err; /* last protocol reply code */
#ifdef WITH_SSL
SSL *ssl; /* SSL handle */
SSL_CTX *ssl_ctx; /* SSL context */
X509 *ssl_cert; /* server certificate */
# if OPENSSL_VERSION_NUMBER < 0x00909000L
SSL_METHOD *ssl_meth; /* SSL method */
# else
const SSL_METHOD *ssl_meth; /* SSL method */
# endif
#endif
int ref; /* reference count */
int is_active;
};
/* Structure used for error message lists */
struct fetcherr {
const int num;
const int cat;
const char *string;
};
/* for fetch_writev */
struct iovec;
void fetch_seterr(struct fetcherr *, int);
void fetch_syserr(void);
void fetch_info(const char *, ...);
int fetch_default_port(const char *);
int fetch_default_proxy_port(const char *);
int fetch_bind(int, int, const char *);
conn_t *fetch_connect(const char *, int, int, int);
conn_t *fetch_reopen(int);
conn_t *fetch_ref(conn_t *);
int fetch_ssl(conn_t *, int);
ssize_t fetch_read(conn_t *, char *, size_t);
int fetch_getln(conn_t *);
ssize_t fetch_write(conn_t *, const char *, size_t);
ssize_t fetch_writev(conn_t *, struct iovec *, int);
int fetch_putln(conn_t *, const char *, size_t);
int fetch_close(conn_t *);
int fetch_add_entry(struct url_list *, struct url *, const char *, int);
int fetch_netrc_auth(struct url *url);
int fetch_no_proxy_match(const char *);
int fetch_urlpath_safe(char);
#define ftp_seterr(n) fetch_seterr(ftp_errlist, n)
#define http_seterr(n) fetch_seterr(http_errlist, n)
#define netdb_seterr(n) fetch_seterr(netdb_errlist, n)
#define url_seterr(n) fetch_seterr(url_errlist, n)
fetchIO *fetchIO_unopen(void *, ssize_t (*)(void *, void *, size_t),
ssize_t (*)(void *, const void *, size_t), void (*)(void *));
/*
* I don't really like exporting http_request() and ftp_request(),
* but the HTTP and FTP code occasionally needs to cross-call
* eachother, and this saves me from adding a lot of special-case code
* to handle those cases.
*
* Note that _*_request() free purl, which is way ugly but saves us a
* whole lot of trouble.
*/
fetchIO *http_request(struct url *, const char *,
struct url_stat *, struct url *, const char *);
fetchIO *ftp_request(struct url *, const char *, const char *,
struct url_stat *, struct url *, const char *);
/*
* Check whether a particular flag is set
*/
#define CHECK_FLAG(x) (flags && strchr(flags, (x)))
#endif

11
lib/fetch/errlist.sh Executable file
View File

@ -0,0 +1,11 @@
#!/bin/sh
# $NetBSD: errlist.sh,v 1.2 2008/10/06 12:58:29 joerg Exp $
printf "static struct fetcherr $1[] = {\n"
while read code type msg; do
[ "${code}" = "#" ] && continue
printf "\t{ ${code}, FETCH_${type}, \"${msg}\" },\n"
done < $3
printf "\t{ -1, FETCH_UNKNOWN, \"Unknown $2 error\" }\n"
printf "};\n"

627
lib/fetch/fetch.c Normal file
View File

@ -0,0 +1,627 @@
/* $NetBSD: fetch.c,v 1.19 2009/08/11 20:48:06 joerg Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav
* Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer
* in this position and unchanged.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD: fetch.c,v 1.41 2007/12/19 00:26:36 des Exp $
*/
#if HAVE_CONFIG_H
#include "config.h"
#endif
#ifndef NETBSD
#include <nbcompat.h>
#endif
#include <ctype.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "fetch.h"
#include "common.h"
auth_t fetchAuthMethod;
int fetchLastErrCode;
char fetchLastErrString[MAXERRSTRING];
int fetchTimeout;
volatile int fetchRestartCalls = 1;
int fetchDebug;
/*** Local data **************************************************************/
/*
* Error messages for parser errors
*/
#define URL_MALFORMED 1
#define URL_BAD_SCHEME 2
#define URL_BAD_PORT 3
static struct fetcherr url_errlist[] = {
{ URL_MALFORMED, FETCH_URL, "Malformed URL" },
{ URL_BAD_SCHEME, FETCH_URL, "Invalid URL scheme" },
{ URL_BAD_PORT, FETCH_URL, "Invalid server port" },
{ -1, FETCH_UNKNOWN, "Unknown parser error" }
};
/*** Public API **************************************************************/
/*
* Select the appropriate protocol for the URL scheme, and return a
* read-only stream connected to the document referenced by the URL.
* Also fill out the struct url_stat.
*/
fetchIO *
fetchXGet(struct url *URL, struct url_stat *us, const char *flags)
{
if (us != NULL) {
us->size = -1;
us->atime = us->mtime = 0;
}
if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
return (fetchXGetFile(URL, us, flags));
else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
return (fetchXGetFTP(URL, us, flags));
else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
return (fetchXGetHTTP(URL, us, flags));
else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
return (fetchXGetHTTP(URL, us, flags));
url_seterr(URL_BAD_SCHEME);
return (NULL);
}
/*
* Select the appropriate protocol for the URL scheme, and return a
* read-only stream connected to the document referenced by the URL.
*/
fetchIO *
fetchGet(struct url *URL, const char *flags)
{
return (fetchXGet(URL, NULL, flags));
}
/*
* Select the appropriate protocol for the URL scheme, and return a
* write-only stream connected to the document referenced by the URL.
*/
fetchIO *
fetchPut(struct url *URL, const char *flags)
{
if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
return (fetchPutFile(URL, flags));
else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
return (fetchPutFTP(URL, flags));
else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
return (fetchPutHTTP(URL, flags));
else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
return (fetchPutHTTP(URL, flags));
url_seterr(URL_BAD_SCHEME);
return (NULL);
}
/*
* Select the appropriate protocol for the URL scheme, and return the
* size of the document referenced by the URL if it exists.
*/
int
fetchStat(struct url *URL, struct url_stat *us, const char *flags)
{
if (us != NULL) {
us->size = -1;
us->atime = us->mtime = 0;
}
if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
return (fetchStatFile(URL, us, flags));
else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
return (fetchStatFTP(URL, us, flags));
else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
return (fetchStatHTTP(URL, us, flags));
else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
return (fetchStatHTTP(URL, us, flags));
url_seterr(URL_BAD_SCHEME);
return (-1);
}
/*
* Select the appropriate protocol for the URL scheme, and return a
* list of files in the directory pointed to by the URL.
*/
int
fetchList(struct url_list *ue, struct url *URL, const char *pattern,
const char *flags)
{
if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
return (fetchListFile(ue, URL, pattern, flags));
else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
return (fetchListFTP(ue, URL, pattern, flags));
else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
return (fetchListHTTP(ue, URL, pattern, flags));
else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
return (fetchListHTTP(ue, URL, pattern, flags));
url_seterr(URL_BAD_SCHEME);
return -1;
}
/*
* Attempt to parse the given URL; if successful, call fetchXGet().
*/
fetchIO *
fetchXGetURL(const char *URL, struct url_stat *us, const char *flags)
{
struct url *u;
fetchIO *f;
if ((u = fetchParseURL(URL)) == NULL)
return (NULL);
f = fetchXGet(u, us, flags);
fetchFreeURL(u);
return (f);
}
/*
* Attempt to parse the given URL; if successful, call fetchGet().
*/
fetchIO *
fetchGetURL(const char *URL, const char *flags)
{
return (fetchXGetURL(URL, NULL, flags));
}
/*
* Attempt to parse the given URL; if successful, call fetchPut().
*/
fetchIO *
fetchPutURL(const char *URL, const char *flags)
{
struct url *u;
fetchIO *f;
if ((u = fetchParseURL(URL)) == NULL)
return (NULL);
f = fetchPut(u, flags);
fetchFreeURL(u);
return (f);
}
/*
* Attempt to parse the given URL; if successful, call fetchStat().
*/
int
fetchStatURL(const char *URL, struct url_stat *us, const char *flags)
{
struct url *u;
int s;
if ((u = fetchParseURL(URL)) == NULL)
return (-1);
s = fetchStat(u, us, flags);
fetchFreeURL(u);
return (s);
}
/*
* Attempt to parse the given URL; if successful, call fetchList().
*/
int
fetchListURL(struct url_list *ue, const char *URL, const char *pattern,
const char *flags)
{
struct url *u;
int rv;
if ((u = fetchParseURL(URL)) == NULL)
return -1;
rv = fetchList(ue, u, pattern, flags);
fetchFreeURL(u);
return rv;
}
/*
* Make a URL
*/
struct url *
fetchMakeURL(const char *scheme, const char *host, int port, const char *doc,
const char *user, const char *pwd)
{
struct url *u;
if (!scheme || (!host && !doc)) {
url_seterr(URL_MALFORMED);
return (NULL);
}
if (port < 0 || port > 65535) {
url_seterr(URL_BAD_PORT);
return (NULL);
}
/* allocate struct url */
if ((u = calloc(1, sizeof(*u))) == NULL) {
fetch_syserr();
return (NULL);
}
if ((u->doc = strdup(doc ? doc : "/")) == NULL) {
fetch_syserr();
free(u);
return (NULL);
}
#define seturl(x) snprintf(u->x, sizeof(u->x), "%s", x)
seturl(scheme);
seturl(host);
seturl(user);
seturl(pwd);
#undef seturl
u->port = port;
return (u);
}
int
fetch_urlpath_safe(char x)
{
if ((x >= '0' && x <= '9') || (x >= 'A' && x <= 'Z') ||
(x >= 'a' && x <= 'z'))
return 1;
switch (x) {
case '$':
case '-':
case '_':
case '.':
case '+':
case '!':
case '*':
case '\'':
case '(':
case ')':
case ',':
/* The following are allowed in segment and path components: */
case '?':
case ':':
case '@':
case '&':
case '=':
case '/':
case ';':
/* If something is already quoted... */
case '%':
return 1;
default:
return 0;
}
}
/*
* Copy an existing URL.
*/
struct url *
fetchCopyURL(const struct url *src)
{
struct url *dst;
char *doc;
/* allocate struct url */
if ((dst = malloc(sizeof(*dst))) == NULL) {
fetch_syserr();
return (NULL);
}
if ((doc = strdup(src->doc)) == NULL) {
fetch_syserr();
free(dst);
return (NULL);
}
*dst = *src;
dst->doc = doc;
return dst;
}
/*
* Split an URL into components. URL syntax is:
* [method:/][/[user[:pwd]@]host[:port]/][document]
* This almost, but not quite, RFC1738 URL syntax.
*/
struct url *
fetchParseURL(const char *URL)
{
const char *p, *q;
struct url *u;
size_t i, count;
int pre_quoted;
/* allocate struct url */
if ((u = calloc(1, sizeof(*u))) == NULL) {
fetch_syserr();
return (NULL);
}
if (*URL == '/') {
pre_quoted = 0;
strcpy(u->scheme, SCHEME_FILE);
p = URL;
goto quote_doc;
}
if (strncmp(URL, "file:", 5) == 0) {
pre_quoted = 1;
strcpy(u->scheme, SCHEME_FILE);
URL += 5;
if (URL[0] != '/' || URL[1] != '/' || URL[2] != '/') {
url_seterr(URL_MALFORMED);
goto ouch;
}
p = URL + 2;
goto quote_doc;
}
if (strncmp(URL, "http:", 5) == 0 ||
strncmp(URL, "https:", 6) == 0) {
pre_quoted = 1;
if (URL[4] == ':') {
strcpy(u->scheme, SCHEME_HTTP);
URL += 5;
} else {
strcpy(u->scheme, SCHEME_HTTPS);
URL += 6;
}
if (URL[0] != '/' || URL[1] != '/') {
url_seterr(URL_MALFORMED);
goto ouch;
}
URL += 2;
p = URL;
goto find_user;
}
if (strncmp(URL, "ftp:", 4) == 0) {
pre_quoted = 1;
strcpy(u->scheme, SCHEME_FTP);
URL += 4;
if (URL[0] != '/' || URL[1] != '/') {
url_seterr(URL_MALFORMED);
goto ouch;
}
URL += 2;
p = URL;
goto find_user;
}
url_seterr(URL_BAD_SCHEME);
goto ouch;
find_user:
p = strpbrk(URL, "/@");
if (p != NULL && *p == '@') {
/* username */
for (q = URL, i = 0; (*q != ':') && (*q != '@'); q++) {
if (i < URL_USERLEN)
u->user[i++] = *q;
}
/* password */
if (*q == ':') {
for (q++, i = 0; (*q != '@'); q++)
if (i < URL_PWDLEN)
u->pwd[i++] = *q;
}
p++;
} else {
p = URL;
}
/* hostname */
#ifdef INET6
if (*p == '[' && (q = strchr(p + 1, ']')) != NULL &&
(*++q == '\0' || *q == '/' || *q == ':')) {
if ((i = q - p - 2) > URL_HOSTLEN)
i = URL_HOSTLEN;
strncpy(u->host, ++p, i);
p = q;
} else
#endif
for (i = 0; *p && (*p != '/') && (*p != ':'); p++)
if (i < URL_HOSTLEN)
u->host[i++] = *p;
/* port */
if (*p == ':') {
for (q = ++p; *q && (*q != '/'); q++)
if (isdigit((unsigned char)*q))
u->port = u->port * 10 + (*q - '0');
else {
/* invalid port */
url_seterr(URL_BAD_PORT);
goto ouch;
}
p = q;
}
/* document */
if (!*p)
p = "/";
quote_doc:
count = 1;
for (i = 0; p[i] != '\0'; ++i) {
if ((!pre_quoted && p[i] == '%') ||
!fetch_urlpath_safe(p[i]))
count += 3;
else
++count;
}
if ((u->doc = malloc(count)) == NULL) {
fetch_syserr();
goto ouch;
}
for (i = 0; *p != '\0'; ++p) {
if ((!pre_quoted && *p == '%') ||
!fetch_urlpath_safe(*p)) {
u->doc[i++] = '%';
if ((unsigned char)*p < 160)
u->doc[i++] = '0' + ((unsigned char)*p) / 16;
else
u->doc[i++] = 'a' - 10 + ((unsigned char)*p) / 16;
if ((unsigned char)*p % 16 < 10)
u->doc[i++] = '0' + ((unsigned char)*p) % 16;
else
u->doc[i++] = 'a' - 10 + ((unsigned char)*p) % 16;
} else
u->doc[i++] = *p;
}
u->doc[i] = '\0';
return (u);
ouch:
free(u);
return (NULL);
}
/*
* Free a URL
*/
void
fetchFreeURL(struct url *u)
{
free(u->doc);
free(u);
}
static char
xdigit2digit(char digit)
{
digit = tolower((unsigned char)digit);
if (digit >= 'a' && digit <= 'f')
digit = digit - 'a' + 10;
else
digit = digit - '0';
return digit;
}
/*
* Unquote whole URL.
* Skips optional parts like query or fragment identifier.
*/
char *
fetchUnquotePath(struct url *url)
{
char *unquoted;
const char *iter;
size_t i;
if ((unquoted = malloc(strlen(url->doc) + 1)) == NULL)
return NULL;
for (i = 0, iter = url->doc; *iter != '\0'; ++iter) {
if (*iter == '#' || *iter == '?')
break;
if (iter[0] != '%' ||
!isxdigit((unsigned char)iter[1]) ||
!isxdigit((unsigned char)iter[2])) {
unquoted[i++] = *iter;
continue;
}
unquoted[i++] = xdigit2digit(iter[1]) * 16 +
xdigit2digit(iter[2]);
iter += 2;
}
unquoted[i] = '\0';
return unquoted;
}
/*
* Extract the file name component of a URL.
*/
char *
fetchUnquoteFilename(struct url *url)
{
char *unquoted, *filename;
const char *last_slash;
if ((unquoted = fetchUnquotePath(url)) == NULL)
return NULL;
if ((last_slash = strrchr(unquoted, '/')) == NULL)
return unquoted;
filename = strdup(last_slash + 1);
free(unquoted);
return filename;
}
char *
fetchStringifyURL(const struct url *url)
{
size_t total;
char *doc;
/* scheme :// user : pwd @ host :port doc */
total = strlen(url->scheme) + 3 + strlen(url->user) + 1 +
strlen(url->pwd) + 1 + strlen(url->host) + 6 + strlen(url->doc) + 1;
if ((doc = malloc(total)) == NULL)
return NULL;
if (url->port != 0)
snprintf(doc, total, "%s%s%s%s%s%s%s:%d%s",
url->scheme,
url->scheme[0] != '\0' ? "://" : "",
url->user,
url->pwd[0] != '\0' ? ":" : "",
url->pwd,
url->user[0] != '\0' || url->pwd[0] != '\0' ? "@" : "",
url->host,
(int)url->port,
url->doc);
else {
snprintf(doc, total, "%s%s%s%s%s%s%s%s",
url->scheme,
url->scheme[0] != '\0' ? "://" : "",
url->user,
url->pwd[0] != '\0' ? ":" : "",
url->pwd,
url->user[0] != '\0' || url->pwd[0] != '\0' ? "@" : "",
url->host,
url->doc);
}
return doc;
}

269
lib/fetch/file.c Normal file
View File

@ -0,0 +1,269 @@
/* $NetBSD: file.c,v 1.15 2009/10/15 12:36:57 joerg Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav
* Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer
* in this position and unchanged.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD: file.c,v 1.18 2007/12/14 10:26:58 des Exp $
*/
#if HAVE_CONFIG_H
#include "config.h"
#endif
#ifndef NETBSD
#include <nbcompat.h>
#endif
#include <sys/stat.h>
#include <dirent.h>
#include <fcntl.h>
#include <fnmatch.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "fetch.h"
#include "common.h"
static int fetch_stat_file(int, struct url_stat *);
static ssize_t
fetchFile_read(void *cookie, void *buf, size_t len)
{
return read(*(int *)cookie, buf, len);
}
static ssize_t
fetchFile_write(void *cookie, const void *buf, size_t len)
{
return write(*(int *)cookie, buf, len);
}
static void
fetchFile_close(void *cookie)
{
int fd = *(int *)cookie;
free(cookie);
close(fd);
}
fetchIO *
fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
{
char *path;
fetchIO *f;
struct url_stat local_us;
int if_modified_since, fd, *cookie;
if_modified_since = CHECK_FLAG('i');
if (if_modified_since && us == NULL)
us = &local_us;
if ((path = fetchUnquotePath(u)) == NULL) {
fetch_syserr();
return NULL;
}
fd = open(path, O_RDONLY);
free(path);
if (fd == -1) {
fetch_syserr();
return NULL;
}
if (us && fetch_stat_file(fd, us) == -1) {
close(fd);
fetch_syserr();
return NULL;
}
if (if_modified_since && u->last_modified > 0 &&
u->last_modified >= us->mtime) {
close(fd);
fetchLastErrCode = FETCH_UNCHANGED;
snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged");
return NULL;
}
if (u->offset && lseek(fd, u->offset, SEEK_SET) == -1) {
close(fd);
fetch_syserr();
return NULL;
}
cookie = malloc(sizeof(int));
if (cookie == NULL) {
close(fd);
fetch_syserr();
return NULL;
}
*cookie = fd;
f = fetchIO_unopen(cookie, fetchFile_read, fetchFile_write, fetchFile_close);
if (f == NULL) {
close(fd);
free(cookie);
}
return f;
}
fetchIO *
fetchGetFile(struct url *u, const char *flags)
{
return (fetchXGetFile(u, NULL, flags));
}
fetchIO *
fetchPutFile(struct url *u, const char *flags)
{
char *path;
fetchIO *f;
int fd, *cookie;
if ((path = fetchUnquotePath(u)) == NULL) {
fetch_syserr();
return NULL;
}
if (CHECK_FLAG('a'))
fd = open(path, O_WRONLY | O_APPEND);
else
fd = open(path, O_WRONLY);
free(path);
if (fd == -1) {
fetch_syserr();
return NULL;
}
if (u->offset && lseek(fd, u->offset, SEEK_SET) == -1) {
close(fd);
fetch_syserr();
return NULL;
}
cookie = malloc(sizeof(int));
if (cookie == NULL) {
close(fd);
fetch_syserr();
return NULL;
}
*cookie = fd;
f = fetchIO_unopen(cookie, fetchFile_read, fetchFile_write, fetchFile_close);
if (f == NULL) {
close(fd);
free(cookie);
}
return f;
}
static int
fetch_stat_file(int fd, struct url_stat *us)
{
struct stat sb;
us->size = -1;
us->atime = us->mtime = 0;
if (fstat(fd, &sb) == -1) {
fetch_syserr();
return (-1);
}
us->size = sb.st_size;
us->atime = sb.st_atime;
us->mtime = sb.st_mtime;
return (0);
}
int
fetchStatFile(struct url *u, struct url_stat *us, const char *flags)
{
char *path;
int fd, rv;
(void)flags;
if ((path = fetchUnquotePath(u)) == NULL) {
fetch_syserr();
return -1;
}
fd = open(path, O_RDONLY);
free(path);
if (fd == -1) {
fetch_syserr();
return -1;
}
rv = fetch_stat_file(fd, us);
close(fd);
return rv;
}
int
fetchListFile(struct url_list *ue, struct url *u, const char *pattern, const char *flags)
{
char *path;
struct dirent *de;
DIR *dir;
int ret;
(void)flags;
if ((path = fetchUnquotePath(u)) == NULL) {
fetch_syserr();
return -1;
}
dir = opendir(path);
free(path);
if (dir == NULL) {
fetch_syserr();
return -1;
}
ret = 0;
while ((de = readdir(dir)) != NULL) {
if (pattern && fnmatch(pattern, de->d_name, 0) != 0)
continue;
ret = fetch_add_entry(ue, u, de->d_name, 0);
if (ret)
break;
}
closedir(dir);
return ret;
}

1299
lib/fetch/ftp.c Normal file
View File

File diff suppressed because it is too large Load Diff

48
lib/fetch/ftp.errors Normal file
View File

@ -0,0 +1,48 @@
# $NetBSD: ftp.errors,v 1.2 2008/10/06 12:58:29 joerg Exp $
# $FreeBSD: ftp.errors,v 1.6 2002/10/30 06:06:16 des Exp $
#
# This list is taken from RFC 959.
# It probably needs a going over.
#
110 OK Restart marker reply
120 TEMP Service ready in a few minutes
125 OK Data connection already open; transfer starting
150 OK File status okay; about to open data connection
200 OK Command okay
202 PROTO Command not implemented, superfluous at this site
211 INFO System status, or system help reply
212 INFO Directory status
213 INFO File status
214 INFO Help message
215 INFO Set system type
220 OK Service ready for new user
221 OK Service closing control connection
225 OK Data connection open; no transfer in progress
226 OK Requested file action successful
227 OK Entering Passive Mode
229 OK Entering Extended Passive Mode
230 OK User logged in, proceed
250 OK Requested file action okay, completed
257 OK File/directory created
331 AUTH User name okay, need password
332 AUTH Need account for login
350 OK Requested file action pending further information
421 DOWN Service not available, closing control connection
425 NETWORK Can't open data connection
426 ABORT Connection closed; transfer aborted
450 UNAVAIL File unavailable (e.g., file busy)
451 SERVER Requested action aborted: local error in processing
452 FULL Insufficient storage space in system
500 PROTO Syntax error, command unrecognized
501 PROTO Syntax error in parameters or arguments
502 PROTO Command not implemented
503 PROTO Bad sequence of commands
504 PROTO Command not implemented for that parameter
530 AUTH Not logged in
532 AUTH Need account for storing files
535 PROTO Bug in MediaHawk Video Kernel FTP server
550 UNAVAIL File unavailable (e.g., file not found, no access)
551 PROTO Requested action aborted. Page type unknown
552 FULL Exceeded storage allocation
553 EXISTS File name not allowed
999 PROTO Protocol error

1460
lib/fetch/http.c Normal file
View File

File diff suppressed because it is too large Load Diff

46
lib/fetch/http.errors Normal file
View File

@ -0,0 +1,46 @@
# $FreeBSD: http.errors,v 1.5 2001/05/23 18:52:02 des Exp $
# $NetBSD: http.errors,v 1.3 2009/02/05 16:59:45 joerg Exp $
#
# This list is taken from RFC 2068.
#
100 OK Continue
101 OK Switching Protocols
200 OK OK
201 OK Created
202 OK Accepted
203 INFO Non-Authoritative Information
204 OK No Content
205 OK Reset Content
206 OK Partial Content
300 MOVED Multiple Choices
301 MOVED Moved Permanently
302 MOVED Moved Temporarily
303 MOVED See Other
304 UNCHANGED Not Modified
305 INFO Use Proxy
307 MOVED Temporary Redirect
400 PROTO Bad Request
401 AUTH Unauthorized
402 AUTH Payment Required
403 AUTH Forbidden
404 UNAVAIL Not Found
405 PROTO Method Not Allowed
406 PROTO Not Acceptable
407 AUTH Proxy Authentication Required
408 TIMEOUT Request Time-out
409 EXISTS Conflict
410 UNAVAIL Gone
411 PROTO Length Required
412 SERVER Precondition Failed
413 PROTO Request Entity Too Large
414 PROTO Request-URI Too Large
415 PROTO Unsupported Media Type
416 UNAVAIL Requested Range Not Satisfiable
417 SERVER Expectation Failed
500 SERVER Internal Server Error
501 PROTO Not Implemented
502 SERVER Bad Gateway
503 TEMP Service Unavailable
504 TIMEOUT Gateway Time-out
505 PROTO HTTP Version not supported
999 PROTO Protocol error