cosmopolitan/net/https/fetch.c

/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8                               :vi │
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2025 Justine Alexandra Roberts Tunney                              │
│                                                                              │
│ Permission to use, copy, modify, and/or distribute this software for         │
│ any purpose with or without fee is hereby granted, provided that the         │
│ above copyright notice and this permission notice appear in all copies.      │
│                                                                              │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL                │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED                │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE             │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL         │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR        │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER               │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
│ PERFORMANCE OF THIS SOFTWARE.                                                │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "net/https/fetch.h"
#include "libc/calls/calls.h"
#include "libc/errno.h"
#include "libc/intrin/kprintf.h"
#include "libc/intrin/likely.h"
#include "libc/macros.h"
#include "libc/serialize.h"
#include "libc/sock/sock.h"
#include "libc/stdio/append.h"
#include "libc/stdio/rand.h"
#include "libc/str/slice.h"
#include "libc/sysv/consts/af.h"
#include "libc/sysv/consts/ipproto.h"
#include "libc/sysv/consts/sock.h"
#include "libc/sysv/consts/sol.h"
#include "libc/sysv/consts/tcp.h"
#include "net/http/http.h"
#include "net/http/url.h"
#include "net/https/https.h"
#include "third_party/mbedtls/aes.h"
#include "third_party/mbedtls/ctr_drbg.h"
#include "third_party/mbedtls/ssl.h"
#include "third_party/musl/netdb.h"

#define FAILNEG1(x)          \
  do {                       \
    if (UNLIKELY((x) == -1)) \
      goto OnFailure;        \
  } while (0)

#define FAILZERO(x)     \
  do {                  \
    if (UNLIKELY(!(x))) \
      goto OnFailure;   \
  } while (0)

#define HasHeader(H)    (!!msg.headers[H].a)
#define HeaderData(H)   (p + msg.headers[H].a)
#define HeaderLength(H) (msg.headers[H].b - msg.headers[H].a)
#define HeaderEqualCase(H, S) \
  SlicesEqualCase(S, strlen(S), HeaderData(H), HeaderLength(H))

struct Fetch {
  int sock;
};

static bool Tune(int fd, int a, int b, int x) {
  if (!b)
    return false;
  return setsockopt(fd, a, b, &x, sizeof(x)) != -1;
}

static int Socket(int family, int type, int protocol) {
  int fd;
  if ((fd = socket(family, type, protocol)) != -1) {
    Tune(fd, SOL_TCP, TCP_FASTOPEN_CONNECT, 1);
    Tune(fd, SOL_TCP, TCP_QUICKACK, 1);
    Tune(fd, SOL_TCP, TCP_NODELAY, 1);
  }
  return fd;
}

static int GetSslEntropy(void *c, unsigned char *p, size_t n) {
  if (getentropy(p, n))
    notpossible;
  return 0;
}

static int TlsSend(void *c, const unsigned char *p, size_t n) {
  ssize_t rc;
  struct Fetch *f = c;
  if ((rc = write(f->sock, p, n)) == -1)
    return -errno;
  return rc;
}

static int TlsRecv(void *c, unsigned char *p, size_t n, uint32_t waitms) {
  ssize_t rc;
  struct Fetch *f = c;
  if ((rc = read(f->sock, p, n)) == -1)
    return -errno;
  return rc;
}

static int PerformFetch(char **response_data, const char *urlarg, int tries) {
  if (tries == 10) {
    errno = ELOOP;
    return -1;
  }

  int retcode;
  uint64_t method = kHttpGet;
  struct Fetch f = {.sock = -1};
  struct Url url = {0};
  char *host = 0;
  char *port = 0;
  char *urldata = 0;
  char *request = 0;
  char *funpath = 0;
  bool usessl = false;
  char *url2 = 0;
  char *p = 0;

  struct HttpMessage msg;
  InitHttpMessage(&msg, kHttpResponse);

  mbedtls_ssl_config conf;
  mbedtls_ssl_context ssl;
  mbedtls_ctr_drbg_context drbg;
  mbedtls_ssl_init(&ssl);
  mbedtls_ctr_drbg_init(&drbg);
  mbedtls_ssl_config_init(&conf);

  if (!(urldata = ParseUrl(urlarg, -1, &url, kUrlPlus)))
    goto BadParameter;
  if (url.scheme.n) {
    if (url.scheme.n == 5 && !memcasecmp(url.scheme.p, "https", 5)) {
      usessl = true;
    } else if (!(url.scheme.n == 4 && !memcasecmp(url.scheme.p, "http", 4))) {
      goto BadParameter;
    }
  }
  if (url.host.n) {
    FAILZERO((host = strndup(url.host.p, url.host.n)));
    if (url.port.n) {
      FAILZERO((port = strndup(url.port.p, url.port.n)));
    } else {
      FAILZERO((port = strdup(usessl ? "443" : "80")));
    }
  } else {
    FAILZERO((host = strdup("127.0.0.1")));
    FAILZERO((port = strdup(usessl ? "443" : "80")));
  }
  url.fragment.p = 0, url.fragment.n = 0;
  url.scheme.p = 0, url.scheme.n = 0;
  url.user.p = 0, url.user.n = 0;
  url.pass.p = 0, url.pass.n = 0;
  url.host.p = 0, url.host.n = 0;
  url.port.p = 0, url.port.n = 0;
  if (!url.path.n || url.path.p[0] != '/') {
    FAILZERO((funpath = malloc(1 + url.path.n)));
    mempcpy(mempcpy(funpath, "/", 1), url.path.p, url.path.n);
    url.path.p = funpath;
    ++url.path.n;
  }

  // build message
  char methodstr[9] = {0};
  WRITE64LE(methodstr, method);
  FAILNEG1(appends(&request, methodstr));
  FAILNEG1(appends(&request, " "));
  char *encoded_url;
  FAILZERO((encoded_url = EncodeUrl(&url, 0)));
  FAILNEG1(appends(&request, encoded_url));
  free(encoded_url);
  FAILNEG1(appends(&request, " HTTP/1.1\r\n"
                             "Connection: close\r\n"));

  // append host header
  FAILNEG1(appends(&request, "Host: "));
  FAILNEG1(appends(&request, host));
  FAILNEG1(appendw(&request, ':'));
  FAILNEG1(appends(&request, port));
  FAILNEG1(appends(&request, "\r\n"));

  // finish headers
  FAILNEG1(appends(&request, "\r\n"));

  // initialize crypto
  if (usessl) {
    if (mbedtls_ctr_drbg_seed(&drbg, GetSslEntropy, 0, "justine", 7))
      goto OutOfMemory;
    mbedtls_ssl_config_defaults(&conf, MBEDTLS_SSL_IS_CLIENT,
                                MBEDTLS_SSL_TRANSPORT_STREAM,
                                MBEDTLS_SSL_PRESET_DEFAULT);
    mbedtls_ssl_conf_authmode(&conf, MBEDTLS_SSL_VERIFY_REQUIRED);
    mbedtls_ssl_conf_ca_chain(&conf, GetSslRoots(), 0);
    mbedtls_ssl_conf_rng(&conf, mbedtls_ctr_drbg_random, &drbg);
    if (mbedtls_ssl_setup(&ssl, &conf))
      goto OutOfMemory;
    if (mbedtls_ssl_set_hostname(&ssl, host))
      goto BadParameter;
  }

  // perform dns lookup
  struct addrinfo *addr;
  struct addrinfo hints = {.ai_family = AF_UNSPEC,
                           .ai_socktype = SOCK_STREAM,
                           .ai_protocol = IPPROTO_TCP,
                           .ai_flags = AI_NUMERICSERV};
  if (getaddrinfo(host, port, &hints, &addr)) {
    errno = ENETUNREACH;
    goto OnFailure;
  }

  // connect to server
  if ((f.sock = Socket(addr->ai_family, addr->ai_socktype,
                       addr->ai_protocol)) == -1 ||
      connect(f.sock, addr->ai_addr, addr->ai_addrlen)) {
    freeaddrinfo(addr);
    goto OnFailure;
  }
  freeaddrinfo(addr);

  // secure connection
  if (usessl) {
    mbedtls_ssl_set_bio(&ssl, &f.sock, TlsSend, 0, TlsRecv);
    if (mbedtls_ssl_handshake(&ssl))
      goto AccessDenied;
  }

  // send message
  ssize_t rc;
  size_t i, n;
  n = appendz(request).i;
  for (i = 0; i < n; i += rc) {
    if (usessl) {
      rc = mbedtls_ssl_write(&ssl, request + i, n - i);
      if (rc <= 0)
        goto OnFailure;
    } else {
      rc = write(f.sock, request + i, n - i);
      if (rc <= 0)
        goto OnFailure;
    }
  }

  // handle response
  int t;
  struct HttpUnchunker u;
  size_t g, hdrlen, paylen;
  FAILZERO((p = malloc((n = 32768))));
  for (hdrlen = paylen = t = i = 0;;) {
    if (i >= n) {
      n = i + 4096;
      char *p2;
      FAILZERO((p2 = realloc(p, n)));
      p = p2;
    }
    if (usessl) {
      if ((rc = mbedtls_ssl_read(&ssl, p + i, n - i)) < 0) {
        if (rc == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) {
          rc = 0;
        } else {
          goto OnFailure;
        }
      }
    } else {
      FAILNEG1((rc = read(f.sock, p + i, n - i)));
    }
    g = rc;
    i += g;
    switch (t) {
      case kHttpClientStateHeaders:
        if ((rc = ParseHttpMessage(&msg, p, i, n)) == -1)
          goto BadMessage;
        if (rc) {
          hdrlen = rc;

          // handle provisional responses
          if (100 <= msg.status && msg.status <= 199) {
            DestroyHttpMessage(&msg);
            InitHttpMessage(&msg, kHttpResponse);
            memmove(p, p + hdrlen, i - hdrlen);
            i -= hdrlen;
            break;
          }

          // handle redirects
          if (300 <= msg.status && msg.status <= 399) {
            if (!HasHeader(kHttpLocation))
              goto BadMessage;
            if (HeaderLength(kHttpLocation) &&
                HeaderData(kHttpLocation)[0] == '/') {
              FAILNEG1(appends(&url2, usessl ? "https://" : "http://"));
              FAILNEG1(appends(&url2, host));
              FAILNEG1(appends(&url2, ":"));
              FAILNEG1(appends(&url2, port));
            }
            FAILNEG1(appendd(&url2, HeaderData(kHttpLocation),
                             HeaderLength(kHttpLocation)));
            close(f.sock);
            f.sock = -1;
            free(p);
            p = 0;
            retcode = PerformFetch(response_data, url2, tries + 1);
            if (retcode == -1 && errno == EINVAL)
              errno = EBADMSG;
            goto OnReturn;
          }

          // handle empty responses
          if (method == kHttpHead || msg.status == 204 || msg.status == 304)
            goto Finished;

          // handle content
          if (HasHeader(kHttpTransferEncoding) &&
              !HeaderEqualCase(kHttpTransferEncoding, "identity")) {
            if (!HeaderEqualCase(kHttpTransferEncoding, "chunked"))
              goto BadMessage;
            t = kHttpClientStateBodyChunked;
            memset(&u, 0, sizeof(u));
            goto Chunked;
          } else if (HasHeader(kHttpContentLength)) {
            if ((rc = ParseContentLength(HeaderData(kHttpContentLength),
                                         HeaderLength(kHttpContentLength))) ==
                -1)
              goto BadMessage;
            t = kHttpClientStateBodyLengthed;
            paylen = rc;
            if (paylen > i - hdrlen) {
              FAILNEG1(appendd(response_data, p + hdrlen, i - hdrlen));
            } else {
              FAILNEG1(appendd(response_data, p + hdrlen, paylen));
              goto Finished;
            }
          } else {
            t = kHttpClientStateBody;
            FAILNEG1(appendd(response_data, p + hdrlen, i - hdrlen));
          }
        }
        break;
      case kHttpClientStateBody:
        FAILNEG1(appendd(response_data, p + i - g, g));
        if (!g)
          goto Finished;
        break;
      case kHttpClientStateBodyLengthed:
        if (i - hdrlen > paylen)
          g = hdrlen + paylen - (i - g);
        FAILNEG1(appendd(response_data, p + i - g, g));
        if (i - hdrlen >= paylen)
          goto Finished;
        break;
      case kHttpClientStateBodyChunked:
      Chunked:
        if ((rc = Unchunk(&u, p + hdrlen, i - hdrlen, &paylen)) == -1)
          goto BadMessage;
        if (rc) {
          FAILNEG1(appendd(response_data, p + hdrlen, paylen));
          goto Finished;
        }
        break;
      default:
        goto BadMessage;
    }
  }

  // we're done
Finished:
  retcode = msg.status;
OnReturn:
  if (f.sock != -1)
    close(f.sock);
  free(p);
  free(url2);
  free(host);
  free(port);
  free(urldata);
  free(request);
  free(funpath);
  free(url.params.p);
  mbedtls_ssl_free(&ssl);
  mbedtls_ssl_config_free(&conf);
  mbedtls_ctr_drbg_free(&drbg);
  DestroyHttpMessage(&msg);
  return retcode;
OnFailure:
  retcode = -1;
  goto OnReturn;
OutOfMemory:
  errno = ENOMEM;
  goto OnFailure;
BadParameter:
  errno = EINVAL;
  goto OnFailure;
BadMessage:
  errno = EBADMSG;
  goto OnFailure;
AccessDenied:
  errno = EPERM;
  goto OnFailure;
}

/**
 * Appends content at URL to `response_data`.
 *
 * - http and https URLs are supported.
 * - Chunked responses will be decoded.
 * - 30x redirect codes will be followed.
 *
 * @return http status code on success, or -1 w/ errno
 * @raise EINVAL if URL parsing failed
 * @raise EPERM if SSL handshake failed
 * @raise ENOMEM if we ran out of memory
 * @raise ENETUNREACH if DNS lookup failed
 * @raise EBADMSG if HTTP message parsing broke
 * @raise ELOOP if more than ten redirects happened
 */
int AppendFetch(char **response_data, const char *url) {
  int sav = errno;
  int res = PerformFetch(response_data, url, 0);
  if (res != -1)
    errno = sav;
  return res;
}