From 90eb1d94337b380c50859defe119feab19aad269 Mon Sep 17 00:00:00 2001 From: Juan RP Date: Sat, 24 Oct 2015 07:52:30 +0200 Subject: [PATCH] libfetch: merge some features from FreeBSD: - Supports HTTP/1.1 308 redirect. - SSLv[23] HTTPS connections are forbidden by default. - TLS client certificate validation thru OpenSSL. - Fixes for user/password encoding, misc. --- NEWS | 7 + lib/fetch/common.c | 552 +++++++++++++++++++++++++++++++++++++++++- lib/fetch/common.h | 12 +- lib/fetch/fetch.c | 58 ++++- lib/fetch/fetch.h | 6 +- lib/fetch/ftp.c | 4 +- lib/fetch/http.c | 56 +++-- lib/fetch/http.errors | 4 +- 8 files changed, 641 insertions(+), 58 deletions(-) diff --git a/NEWS b/NEWS index 4b2e7eb5..86f4d613 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,12 @@ xbps-0.48 (???): + * libfetch: merge some features from FreeBSD: + + - Supports HTTP/1.1 308 redirect. + - SSLv[23] HTTPS connections are forbidden by default. + - TLS client certificate validation thru OpenSSL. + - Fixes for user/password encoding, misc. + * lixbps: use a sane umask if the pkgdb file needs to created for the first time. Thanks to Wolfgang Draxinger (https://github.com/voidlinux/xbps/pull/108). diff --git a/lib/fetch/common.c b/lib/fetch/common.c index ea50bd62..9640799f 100644 --- a/lib/fetch/common.c +++ b/lib/fetch/common.c @@ -1,7 +1,9 @@ +/* $FreeBSD: rev 288217 $ */ /* $NetBSD: common.c,v 1.29 2014/01/08 20:25:34 joerg Exp $ */ /*- - * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav + * Copyright (c) 1998-2014 Dag-Erling Smorgrav * Copyright (c) 2008, 2010 Joerg Sonnenberger + * Copyright (c) 2013 Michael Gmelin * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -26,8 +28,6 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * $FreeBSD: common.c,v 1.53 2007/12/19 00:26:36 des Exp $ */ #include "compat.h" @@ -58,6 +58,10 @@ #include #endif +#ifdef WITH_SSL +#include +#endif + #include #include "fetch.h" @@ -438,6 +442,495 @@ fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *)) pthread_mutex_unlock(&cache_mtx); } + +#ifdef WITH_SSL +/* + * Find the first occurrence of find in s, where the search is limited to the + * first slen characters of s. + */ +static char * +strnstr(const char *s, const char *find, size_t slen) +{ + char c, sc; + size_t len; + + if ((c = *find++) != '\0') { + len = strlen(find); + do { + do { + if (slen-- < 1 || (sc = *s++) == '\0') + return (NULL); + } while (sc != c); + if (len > slen) + return (NULL); + } while (strncmp(s, find, len) != 0); + s--; + } + return ((char *)__UNCONST(s)); +} + +/* + * Convert characters A-Z to lowercase (intentionally avoid any locale + * specific conversions). + */ +static char +fetch_ssl_tolower(char in) +{ + if (in >= 'A' && in <= 'Z') + return (in + 32); + else + return (in); +} + +/* + * isalpha implementation that intentionally avoids any locale specific + * conversions. + */ +static int +fetch_ssl_isalpha(char in) +{ + return ((in >= 'A' && in <= 'Z') || (in >= 'a' && in <= 'z')); +} + +/* + * Check if passed hostnames a and b are equal. + */ +static int +fetch_ssl_hname_equal(const char *a, size_t alen, const char *b, + size_t blen) +{ + size_t i; + + if (alen != blen) + return (0); + for (i = 0; i < alen; ++i) { + if (fetch_ssl_tolower(a[i]) != fetch_ssl_tolower(b[i])) + return (0); + } + return (1); +} + +/* + * Check if domain label is traditional, meaning that only A-Z, a-z, 0-9 + * and '-' (hyphen) are allowed. Hyphens have to be surrounded by alpha- + * numeric characters. Double hyphens (like they're found in IDN a-labels + * 'xn--') are not allowed. Empty labels are invalid. + */ +static int +fetch_ssl_is_trad_domain_label(const char *l, size_t len, int wcok) +{ + size_t i; + + if (!len || l[0] == '-' || l[len-1] == '-') + return (0); + for (i = 0; i < len; ++i) { + if (!isdigit(l[i]) && + !fetch_ssl_isalpha(l[i]) && + !(l[i] == '*' && wcok) && + !(l[i] == '-' && l[i - 1] != '-')) + return (0); + } + return (1); +} + +/* + * Check if host name consists only of numbers. This might indicate an IP + * address, which is not a good idea for CN wildcard comparison. + */ +static int +fetch_ssl_hname_is_only_numbers(const char *hostname, size_t len) +{ + size_t i; + + for (i = 0; i < len; ++i) { + if (!((hostname[i] >= '0' && hostname[i] <= '9') || + hostname[i] == '.')) + return (0); + } + return (1); +} + +/* + * Check if the host name h passed matches the pattern passed in m which + * is usually part of subjectAltName or CN of a certificate presented to + * the client. This includes wildcard matching. The algorithm is based on + * RFC6125, sections 6.4.3 and 7.2, which clarifies RFC2818 and RFC3280. + */ +static int +fetch_ssl_hname_match(const char *h, size_t hlen, const char *m, + size_t mlen) +{ + int delta, hdotidx, mdot1idx, wcidx; + const char *hdot, *mdot1, *mdot2; + const char *wc; /* wildcard */ + + if (!(h && *h && m && *m)) + return (0); + if ((wc = strnstr(m, "*", mlen)) == NULL) + return (fetch_ssl_hname_equal(h, hlen, m, mlen)); + wcidx = wc - m; + /* hostname should not be just dots and numbers */ + if (fetch_ssl_hname_is_only_numbers(h, hlen)) + return (0); + /* only one wildcard allowed in pattern */ + if (strnstr(wc + 1, "*", mlen - wcidx - 1) != NULL) + return (0); + /* + * there must be at least two more domain labels and + * wildcard has to be in the leftmost label (RFC6125) + */ + mdot1 = strnstr(m, ".", mlen); + if (mdot1 == NULL || mdot1 < wc || (mlen - (mdot1 - m)) < 4) + return (0); + mdot1idx = mdot1 - m; + mdot2 = strnstr(mdot1 + 1, ".", mlen - mdot1idx - 1); + if (mdot2 == NULL || (mlen - (mdot2 - m)) < 2) + return (0); + /* hostname must contain a dot and not be the 1st char */ + hdot = strnstr(h, ".", hlen); + if (hdot == NULL || hdot == h) + return (0); + hdotidx = hdot - h; + /* + * host part of hostname must be at least as long as + * pattern it's supposed to match + */ + if (hdotidx < mdot1idx) + return (0); + /* + * don't allow wildcards in non-traditional domain names + * (IDN, A-label, U-label...) + */ + if (!fetch_ssl_is_trad_domain_label(h, hdotidx, 0) || + !fetch_ssl_is_trad_domain_label(m, mdot1idx, 1)) + return (0); + /* match domain part (part after first dot) */ + if (!fetch_ssl_hname_equal(hdot, hlen - hdotidx, mdot1, + mlen - mdot1idx)) + return (0); + /* match part left of wildcard */ + if (!fetch_ssl_hname_equal(h, wcidx, m, wcidx)) + return (0); + /* match part right of wildcard */ + delta = mdot1idx - wcidx - 1; + if (!fetch_ssl_hname_equal(hdot - delta, delta, + mdot1 - delta, delta)) + return (0); + /* all tests succeded, it's a match */ + return (1); +} + +/* + * Get numeric host address info - returns NULL if host was not an IP + * address. The caller is responsible for deallocation using + * freeaddrinfo(3). + */ +static struct addrinfo * +fetch_ssl_get_numeric_addrinfo(const char *hostname, size_t len) +{ + struct addrinfo hints, *res; + char *host; + + host = (char *)malloc(len + 1); + memcpy(host, hostname, len); + host[len] = '\0'; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = 0; + hints.ai_flags = AI_NUMERICHOST; + /* port is not relevant for this purpose */ + if (getaddrinfo(host, "443", &hints, &res) != 0) + return NULL; + free(host); + return res; +} + +/* + * Compare ip address in addrinfo with address passes. + */ +static int +fetch_ssl_ipaddr_match_bin(const struct addrinfo *lhost, const char *rhost, + size_t rhostlen) +{ + const void *left; + + if (lhost->ai_family == AF_INET && rhostlen == 4) { + left = (void *)&((struct sockaddr_in*)(void *) + lhost->ai_addr)->sin_addr.s_addr; +#ifdef INET6 + } else if (lhost->ai_family == AF_INET6 && rhostlen == 16) { + left = (void *)&((struct sockaddr_in6 *)(void *) + lhost->ai_addr)->sin6_addr; +#endif + } else + return (0); + return (!memcmp(left, (const void *)rhost, rhostlen) ? 1 : 0); +} + +/* + * Compare ip address in addrinfo with host passed. If host is not an IP + * address, comparison will fail. + */ +static int +fetch_ssl_ipaddr_match(const struct addrinfo *laddr, const char *r, + size_t rlen) +{ + struct addrinfo *raddr; + int ret; + char *rip; + + ret = 0; + if ((raddr = fetch_ssl_get_numeric_addrinfo(r, rlen)) == NULL) + return 0; /* not a numeric host */ + + if (laddr->ai_family == raddr->ai_family) { + if (laddr->ai_family == AF_INET) { + rip = (char *)&((struct sockaddr_in *)(void *) + raddr->ai_addr)->sin_addr.s_addr; + ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 4); +#ifdef INET6 + } else if (laddr->ai_family == AF_INET6) { + rip = (char *)&((struct sockaddr_in6 *)(void *) + raddr->ai_addr)->sin6_addr; + ret = fetch_ssl_ipaddr_match_bin(laddr, rip, 16); +#endif + } + } + freeaddrinfo(raddr); + return (ret); +} + +/* + * Verify server certificate by subjectAltName. + */ +static int +fetch_ssl_verify_altname(STACK_OF(GENERAL_NAME) *altnames, + const char *host, struct addrinfo *ip) +{ + const GENERAL_NAME *name; + size_t nslen; + int i; + const char *ns; + + for (i = 0; i < sk_GENERAL_NAME_num(altnames); ++i) { + name = sk_GENERAL_NAME_value(altnames, i); + ns = (const char *)ASN1_STRING_data(name->d.ia5); + nslen = (size_t)ASN1_STRING_length(name->d.ia5); + + if (name->type == GEN_DNS && ip == NULL && + fetch_ssl_hname_match(host, strlen(host), ns, nslen)) + return (1); + else if (name->type == GEN_IPADD && ip != NULL && + fetch_ssl_ipaddr_match_bin(ip, ns, nslen)) + return (1); + } + return (0); +} + +/* + * Verify server certificate by CN. + */ +static int +fetch_ssl_verify_cn(X509_NAME *subject, const char *host, + struct addrinfo *ip) +{ + ASN1_STRING *namedata; + X509_NAME_ENTRY *nameentry; + int cnlen, lastpos, loc, ret; + unsigned char *cn; + + ret = 0; + lastpos = -1; + loc = -1; + cn = NULL; + /* get most specific CN (last entry in list) and compare */ + while ((lastpos = X509_NAME_get_index_by_NID(subject, + NID_commonName, lastpos)) != -1) + loc = lastpos; + + if (loc > -1) { + nameentry = X509_NAME_get_entry(subject, loc); + namedata = X509_NAME_ENTRY_get_data(nameentry); + cnlen = ASN1_STRING_to_UTF8(&cn, namedata); + if (ip == NULL && + fetch_ssl_hname_match(host, strlen(host), (const char *)cn, cnlen)) + ret = 1; + else if (ip != NULL && fetch_ssl_ipaddr_match(ip, (const char *)cn, cnlen)) + ret = 1; + OPENSSL_free(cn); + } + return (ret); +} + +/* + * Verify that server certificate subjectAltName/CN matches + * hostname. First check, if there are alternative subject names. If yes, + * those have to match. Only if those don't exist it falls back to + * checking the subject's CN. + */ +static int +fetch_ssl_verify_hname(X509 *cert, const char *host) +{ + struct addrinfo *ip; + STACK_OF(GENERAL_NAME) *altnames; + X509_NAME *subject; + int ret; + + ret = 0; + ip = fetch_ssl_get_numeric_addrinfo(host, strlen(host)); + altnames = X509_get_ext_d2i(cert, NID_subject_alt_name, + NULL, NULL); + + if (altnames != NULL) { + ret = fetch_ssl_verify_altname(altnames, host, ip); + } else { + subject = X509_get_subject_name(cert); + if (subject != NULL) + ret = fetch_ssl_verify_cn(subject, host, ip); + } + + if (ip != NULL) + freeaddrinfo(ip); + if (altnames != NULL) + GENERAL_NAMES_free(altnames); + return (ret); +} + +/* + * Configure transport security layer based on environment. + */ +static void +fetch_ssl_setup_transport_layer(SSL_CTX *ctx, int verbose) +{ + long ssl_ctx_options; + + ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_TICKET; + if (getenv("SSL_ALLOW_SSL3") == NULL) + ssl_ctx_options |= SSL_OP_NO_SSLv3; + if (getenv("SSL_NO_TLS1") != NULL) + ssl_ctx_options |= SSL_OP_NO_TLSv1; + if (getenv("SSL_NO_TLS1_1") != NULL) + ssl_ctx_options |= SSL_OP_NO_TLSv1_1; + if (getenv("SSL_NO_TLS1_2") != NULL) + ssl_ctx_options |= SSL_OP_NO_TLSv1_2; + if (verbose) + fetch_info("SSL options: %lx", ssl_ctx_options); + SSL_CTX_set_options(ctx, ssl_ctx_options); +} + + +/* + * Configure peer verification based on environment. + */ +static int +fetch_ssl_setup_peer_verification(SSL_CTX *ctx, int verbose) +{ + X509_LOOKUP *crl_lookup; + X509_STORE *crl_store; + const char *ca_cert_file, *ca_cert_path, *crl_file; + + if (getenv("SSL_NO_VERIFY_PEER") == NULL) { + ca_cert_file = getenv("SSL_CA_CERT_FILE") != NULL ? + getenv("SSL_CA_CERT_FILE") : "/etc/ssl/cert.pem"; + ca_cert_path = getenv("SSL_CA_CERT_PATH"); + if (verbose) { + fetch_info("Peer verification enabled"); + if (ca_cert_file != NULL) + fetch_info("Using CA cert file: %s", + ca_cert_file); + if (ca_cert_path != NULL) + fetch_info("Using CA cert path: %s", + ca_cert_path); + } + SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, + fetch_ssl_cb_verify_crt); + SSL_CTX_load_verify_locations(ctx, ca_cert_file, + ca_cert_path); + if ((crl_file = getenv("SSL_CRL_FILE")) != NULL) { + if (verbose) + fetch_info("Using CRL file: %s", crl_file); + + crl_store = SSL_CTX_get_cert_store(ctx); + crl_lookup = X509_STORE_add_lookup(crl_store, + X509_LOOKUP_file()); + if (crl_lookup == NULL || + !X509_load_crl_file(crl_lookup, crl_file, + X509_FILETYPE_PEM)) { + fprintf(stderr, + "Could not load CRL file %s\n", + crl_file); + return (0); + } + X509_STORE_set_flags(crl_store, + X509_V_FLAG_CRL_CHECK | + X509_V_FLAG_CRL_CHECK_ALL); + } + } + return (1); +} + +/* + * Configure client certificate based on environment. + */ +static int +fetch_ssl_setup_client_certificate(SSL_CTX *ctx, int verbose) +{ + const char *client_cert_file, *client_key_file; + + if ((client_cert_file = getenv("SSL_CLIENT_CERT_FILE")) != NULL) { + client_key_file = getenv("SSL_CLIENT_KEY_FILE") != NULL ? + getenv("SSL_CLIENT_KEY_FILE") : client_cert_file; + if (verbose) { + fetch_info("Using client cert file: %s", + client_cert_file); + fetch_info("Using client key file: %s", + client_key_file); + } + if (SSL_CTX_use_certificate_chain_file(ctx, + client_cert_file) != 1) { + fprintf(stderr, + "Could not load client certificate %s\n", + client_cert_file); + return (0); + } + if (SSL_CTX_use_PrivateKey_file(ctx, client_key_file, + SSL_FILETYPE_PEM) != 1) { + fprintf(stderr, + "Could not load client key %s\n", + client_key_file); + return (0); + } + } + return (1); +} + +/* + * Callback for SSL certificate verification, this is called on server + * cert verification. It takes no decision, but informs the user in case + * verification failed. + */ +int +fetch_ssl_cb_verify_crt(int verified, X509_STORE_CTX *ctx) +{ + X509 *crt; + X509_NAME *name; + char *str; + + str = NULL; + if (!verified) { + if ((crt = X509_STORE_CTX_get_current_cert(ctx)) != NULL && + (name = X509_get_subject_name(crt)) != NULL) + str = X509_NAME_oneline(name, 0, 0); + fprintf(stderr, "Certificate verification failed for %s\n", + str != NULL ? str : "no relevant certificate"); + OPENSSL_free(str); + } + return (verified); +} + +#endif + /* * Enable SSL on a connection. */ @@ -447,6 +940,8 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose) #ifdef WITH_SSL int ret; + X509_NAME *name; + char *str; /* Init the SSL library and context */ if (!SSL_library_init()){ @@ -460,8 +955,14 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose) conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth); SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY); + fetch_ssl_setup_transport_layer(conn->ssl_ctx, verbose); + if (!fetch_ssl_setup_peer_verification(conn->ssl_ctx, verbose)) + return (-1); + if (!fetch_ssl_setup_client_certificate(conn->ssl_ctx, verbose)) + return (-1); + conn->ssl = SSL_new(conn->ssl_ctx); - if (conn->ssl == NULL){ + if (conn->ssl == NULL) { fprintf(stderr, "SSL context creation failed\n"); return (-1); } @@ -483,21 +984,36 @@ fetch_ssl(conn_t *conn, const struct url *URL, int verbose) return (-1); } - if (verbose) { - X509_NAME *name; - char *str; + conn->ssl_cert = SSL_get_peer_certificate(conn->ssl); - fprintf(stderr, "SSL connection established using %s\n", - SSL_get_cipher(conn->ssl)); + if (conn->ssl_cert == NULL) { + fprintf(stderr, "No server SSL certificate\n"); + return (-1); + } + + if (getenv("SSL_NO_VERIFY_HOSTNAME") == NULL) { + if (verbose) + fetch_info("Verify hostname"); + if (!fetch_ssl_verify_hname(conn->ssl_cert, URL->host)) { + fprintf(stderr, + "SSL certificate subject doesn't match host %s\n", + URL->host); + return (-1); + } + } + + if (verbose) { + fetch_info("%s connection established using %s", + SSL_get_version(conn->ssl), SSL_get_cipher(conn->ssl)); conn->ssl_cert = SSL_get_peer_certificate(conn->ssl); name = X509_get_subject_name(conn->ssl_cert); str = X509_NAME_oneline(name, 0, 0); - printf("Certificate subject: %s\n", str); - free(str); + fetch_info("Certificate subject: %s", str); + OPENSSL_free(str); name = X509_get_issuer_name(conn->ssl_cert); str = X509_NAME_oneline(name, 0, 0); - printf("Certificate issuer: %s\n", str); - free(str); + fetch_info("Certificate issuer: %s", str); + OPENSSL_free(str); } return (0); @@ -744,7 +1260,17 @@ fetch_close(conn_t *conn) #ifdef WITH_SSL if (conn->ssl) { SSL_shutdown(conn->ssl); + SSL_set_connect_state(conn->ssl); SSL_free(conn->ssl); + conn->ssl = NULL; + } + if (conn->ssl_ctx) { + SSL_CTX_free(conn->ssl_ctx); + conn->ssl_ctx = NULL; + } + if (conn->ssl_cert) { + X509_free(conn->ssl_cert); + conn->ssl_cert = NULL; } #endif ret = close(conn->sd); diff --git a/lib/fetch/common.h b/lib/fetch/common.h index 2c363217..4c261577 100644 --- a/lib/fetch/common.h +++ b/lib/fetch/common.h @@ -1,6 +1,7 @@ +/* $FreeBSD: rev 267133 $ */ /* $NetBSD: common.h,v 1.23 2014/01/08 20:25:34 joerg Exp $ */ /*- - * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav + * Copyright (c) 1998-2014 Dag-Erling Smorgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -25,8 +26,6 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * $FreeBSD: common.h,v 1.30 2007/12/18 11:03:07 des Exp $ */ #ifndef _COMMON_H_INCLUDED @@ -106,6 +105,9 @@ conn_t *fetch_cache_get(const struct url *, int); void fetch_cache_put(conn_t *, int (*)(conn_t *)); conn_t *fetch_connect(struct url *, int, int); conn_t *fetch_reopen(int); +#ifdef WITH_SSL +int fetch_ssl_cb_verify_crt(int, X509_STORE_CTX*); +#endif int fetch_ssl(conn_t *, const struct url *, int); ssize_t fetch_read(conn_t *, char *, size_t); int fetch_getln(conn_t *); @@ -144,4 +146,8 @@ fetchIO *ftp_request(struct url *, const char *, const char *, */ #define CHECK_FLAG(x) (flags && strchr(flags, (x))) +#ifndef __UNCONST +#define __UNCONST(a) ((void *)(unsigned long)(const void *)(a)) +#endif + #endif diff --git a/lib/fetch/fetch.c b/lib/fetch/fetch.c index a8190b62..acdb716b 100644 --- a/lib/fetch/fetch.c +++ b/lib/fetch/fetch.c @@ -1,3 +1,4 @@ +/* $FreeBSD: rev 252375 $ */ /* $NetBSD: fetch.c,v 1.19 2009/08/11 20:48:06 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørav @@ -26,8 +27,6 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * $FreeBSD: fetch.c,v 1.41 2007/12/19 00:26:36 des Exp $ */ #include "compat.h" @@ -292,6 +291,48 @@ fetchMakeURL(const char *scheme, const char *host, int port, const char *doc, return (u); } +/* + * Return value of the given hex digit. + */ +static int +fetch_hexval(char ch) +{ + if (ch >= '0' && ch <= '9') + return (ch - '0'); + else if (ch >= 'a' && ch <= 'f') + return (ch - 'a' + 10); + else if (ch >= 'A' && ch <= 'F') + return (ch - 'A' + 10); + return (-1); +} + +/* + * Decode percent-encoded URL component from src into dst, stopping at end + * of string, or at @ or : separators. Returns a pointer to the unhandled + * part of the input string (null terminator, @, or :). No terminator is + * written to dst (it is the caller's responsibility). + */ +static const char * +fetch_pctdecode(char *dst, const char *src, size_t dlen) +{ + int d1, d2; + char c; + const char *s; + + for (s = src; *s != '\0' && *s != '@' && *s != ':'; s++) { + if (s[0] == '%' && (d1 = fetch_hexval(s[1])) >= 0 && + (d2 = fetch_hexval(s[2])) >= 0 && (d1 > 0 || d2 > 0)) { + c = d1 << 4 | d2; + s += 2; + } else { + c = *s; + } + if (dlen-- > 0) + *dst++ = c; + } + return (s); +} + int fetch_urlpath_safe(char x) { @@ -426,17 +467,10 @@ find_user: p = strpbrk(URL, "/@"); if (p != NULL && *p == '@') { /* username */ - for (q = URL, i = 0; (*q != ':') && (*q != '@'); q++) { - if (i < URL_USERLEN) - u->user[i++] = *q; - } - + q = fetch_pctdecode(u->user, URL, URL_USERLEN); /* password */ - if (*q == ':') { - for (q++, i = 0; (*q != '@'); q++) - if (i < URL_PWDLEN) - u->pwd[i++] = *q; - } + if (*q == ':') + q = fetch_pctdecode(u->pwd, q + 1, URL_PWDLEN); p++; } else { diff --git a/lib/fetch/fetch.h b/lib/fetch/fetch.h index 30754864..c5728581 100644 --- a/lib/fetch/fetch.h +++ b/lib/fetch/fetch.h @@ -1,6 +1,6 @@ /* $NetBSD: fetch.h,v 1.16 2010/01/22 13:21:09 joerg Exp $ */ /*- - * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav + * Copyright (c) 1998-2014 Dag-Erling Smorgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -25,8 +25,6 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * $FreeBSD: fetch.h,v 1.26 2004/09/21 18:35:20 des Exp $ */ #ifndef _FETCH_H_INCLUDED @@ -36,7 +34,7 @@ #include #include -#define _LIBFETCH_VER "libfetch/2.0" +#define _LIBFETCH_VER "xbps/2.0" #define URL_HOSTLEN 255 #define URL_SCHEMELEN 16 diff --git a/lib/fetch/ftp.c b/lib/fetch/ftp.c index 51fe92a7..e7ba9039 100644 --- a/lib/fetch/ftp.c +++ b/lib/fetch/ftp.c @@ -1,6 +1,6 @@ /* $NetBSD: ftp.c,v 1.46 2014/06/11 13:12:12 joerg Exp $ */ /*- - * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav + * Copyright (c) 1998-2004 Dag-Erling Smorgrav * Copyright (c) 2008, 2009, 2010 Joerg Sonnenberger * All rights reserved. * @@ -42,7 +42,7 @@ * * Major Changelog: * - * Dag-Erling CoýÅan Smgrav + * Dag-Erling Smograv * 9 Jun 1998 * * Incorporated into libfetch diff --git a/lib/fetch/http.c b/lib/fetch/http.c index 848ed1a0..a027c39d 100644 --- a/lib/fetch/http.c +++ b/lib/fetch/http.c @@ -1,6 +1,7 @@ +/* $FreeBSD: rev 267127 $ */ /* $NetBSD: http.c,v 1.37 2014/06/11 13:12:12 joerg Exp $ */ /*- - * Copyright (c) 2000-2004 Dag-Erling CoýÅan Smgrav + * Copyright (c) 2000-2014 Dag-Erling Smorgrav * Copyright (c) 2003 Thomas Klausner * Copyright (c) 2008, 2009 Joerg Sonnenberger * All rights reserved. @@ -27,8 +28,6 @@ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * $FreeBSD: http.c,v 1.83 2008/02/06 11:39:55 des Exp $ */ /* @@ -63,12 +62,7 @@ * SUCH DAMAGE. */ -#if defined(__linux__) || defined(__MINT__) || defined(__FreeBSD_kernel__) -/* Keep this down to Linux or MiNT, it can create surprises elsewhere. */ -/* - __FreeBSD_kernel__ is defined for GNU/kFreeBSD. - See http://glibc-bsd.alioth.debian.org/porting/PORTING . -*/ +#if defined(__linux__) #define _GNU_SOURCE #endif @@ -106,7 +100,9 @@ #define HTTP_MOVED_TEMP 302 #define HTTP_SEE_OTHER 303 #define HTTP_NOT_MODIFIED 304 +#define HTTP_USE_PROXY 305 #define HTTP_TEMP_REDIRECT 307 +#define HTTP_PERM_REDIRECT 308 #define HTTP_NEED_AUTH 401 #define HTTP_NEED_PROXY_AUTH 407 #define HTTP_BAD_RANGE 416 @@ -115,6 +111,7 @@ #define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \ || (xyz) == HTTP_MOVED_TEMP \ || (xyz) == HTTP_TEMP_REDIRECT \ + || (xyz) == HTTP_USE_PROXY \ || (xyz) == HTTP_SEE_OTHER) #define HTTP_ERROR(xyz) ((xyz) > 400 && (xyz) < 599) @@ -517,6 +514,12 @@ http_parse_mtime(const char *p, time_t *mtime) locale[sizeof(locale)-1] = '\0'; setlocale(LC_TIME, "C"); r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm); + /* + * Some proxies use UTC in response, but it should still be + * parsed. RFC2616 states GMT and UTC are exactly equal for HTTP. + */ + if (r == NULL) + r = strptime(p, "%a, %d %b %Y %H:%M:%S UTC", &tm); /* XXX should add support for date-2 and date-3 */ setlocale(LC_TIME, locale); if (r == NULL) @@ -698,6 +701,7 @@ http_authorize(conn_t *conn, const char *hdr, const char *p) static conn_t * http_connect(struct url *URL, struct url *purl, const char *flags, int *cached) { + struct url *curl; conn_t *conn; int af, verbose; #ifdef TCP_NOPUSH @@ -718,22 +722,25 @@ http_connect(struct url *URL, struct url *purl, const char *flags, int *cached) af = AF_INET6; #endif - if (purl && strcasecmp(URL->scheme, SCHEME_HTTPS) != 0) { - URL = purl; - } else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) { - /* can't talk http to an ftp server */ - /* XXX should set an error code */ - return (NULL); - } + curl = (purl != NULL) ? purl : URL; - if ((conn = fetch_cache_get(URL, af)) != NULL) { + if ((conn = fetch_cache_get(curl, af)) != NULL) { *cached = 1; return (conn); } - if ((conn = fetch_connect(URL, af, verbose)) == NULL) + if ((conn = fetch_connect(curl, af, verbose)) == NULL) /* fetch_connect() has already set an error code */ return (NULL); + if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) { + http_cmd(conn, "CONNECT %s:%d HTTP/1.1", + URL->host, URL->port); + if (http_get_reply(conn) != HTTP_OK) { + fetch_close(conn); + return (NULL); + } + http_get_reply(conn); + } if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && fetch_ssl(conn, URL, verbose) == -1) { fetch_close(conn); @@ -888,7 +895,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us, if (verbose) fetch_info("requesting %s://%s%s", url->scheme, host, url->doc); - if (purl) { + if (purl && strcasecmp(URL->scheme, SCHEME_HTTPS) != 0) { http_cmd(conn, "%s %s://%s%s HTTP/1.1\r\n", op, url->scheme, host, url->doc); } else { @@ -933,10 +940,14 @@ http_request(struct url *URL, const char *op, struct url_stat *us, else http_cmd(conn, "Referer: %s\r\n", p); } - if ((p = getenv("HTTP_USER_AGENT")) != NULL && *p != '\0') - http_cmd(conn, "User-Agent: %s\r\n", p); - else + if ((p = getenv("HTTP_USER_AGENT")) != NULL) { + /* no User-Agent if defined but empty */ + if (*p != '\0') + http_cmd(conn, "User-Agent: %s\r\n", p); + } else { + /* default User-Agent */ http_cmd(conn, "User-Agent: %s\r\n", _LIBFETCH_VER); + } /* * Some servers returns 406 (Not Acceptable) if the Accept field is not @@ -975,6 +986,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us, case HTTP_MOVED_PERM: case HTTP_MOVED_TEMP: case HTTP_SEE_OTHER: + case HTTP_USE_PROXY: /* * Not so fine, but we still have to read the * headers to get the new location. diff --git a/lib/fetch/http.errors b/lib/fetch/http.errors index 004aac27..d1e1cd34 100644 --- a/lib/fetch/http.errors +++ b/lib/fetch/http.errors @@ -1,5 +1,4 @@ -# $FreeBSD: http.errors,v 1.5 2001/05/23 18:52:02 des Exp $ -# $NetBSD: http.errors,v 1.3 2009/02/05 16:59:45 joerg Exp $ +# $FreeBSD$ revision 241840 # # This list is taken from RFC 2068. # @@ -19,6 +18,7 @@ 304 UNCHANGED Not Modified 305 INFO Use Proxy 307 MOVED Temporary Redirect +308 MOVED Permanent Redirect 400 PROTO Bad Request 401 AUTH Unauthorized 402 AUTH Payment Required