Compare commits

..
Author SHA1 Message Date
Justine Tunney 6a04bc3318 Release redbean 2.0.18 2022-09-04 06:57:59 -07:00
Justine Tunney 8dd4ec68d0 Add more missing C / C++ headers 2022-09-04 04:53:52 -07:00
Justine Tunney b9dc74b672 Do some work on unbuffer command 2022-09-04 02:25:34 -07:00
Justine Tunney e1590e1e38 Remove accidental comment 2022-09-04 00:41:09 -07:00
tkchiaandtkchia 8569704c1d Stop APE bare metal loader from reading beyond program image end (#574)
This allows e.g. `qemu-system-x86_64 -s o/examples/hello.com
-serial stdio` to work without having to add extra padding to
the end of the `hello.com` "disk image".

(The sector count computation is divided among two instructions
in the assembly code.  This is done on purpose, to prevent an
ASCII 0x27 (single quote) byte from appearing in the bare
metal loader code, which will break the shell script loader.
There is probably a better way.)

Co-authored-by: tkchia <[email protected]>
2022-09-04 00:19:08 -07:00
Gavin Hayes 9c017c98d3 Add sysconf to unistd.h (#575) 2022-09-04 00:10:17 -07:00
Gavin Hayes 74879bf067 Add pollfd to poll.h (#576) 2022-09-04 00:09:50 -07:00
Paul Kulchenko 8a3d8497e3 Update README to add a note about binfmt-related errors (#587) 2022-09-04 00:09:24 -07:00
Justine Tunney 3c00d8c29c Fix TLS linker warning
Fixes #565
Closes #577
Co-authored-by: tkchia <[email protected]>
2022-09-04 00:05:30 -07:00
Gavin Hayes 1ef955c33b Improve zipos path handling (#579)
This change adds opendir, readdir, and stat support for the /zip/ root,
as well as directories not explicitly encoded in the zip file.
2022-09-03 22:50:23 -07:00
Gavin Hayes 494d74271b Fix inet_ntop(AF_INET6) buffer length (#582) 2022-09-03 22:46:45 -07:00
Ilya Brin 12212894eb Highlight GDB syntax in README.md (#560) 2022-09-03 22:19:55 -07:00
Justine Tunney a4522fba8d Fix build 2022-09-03 21:10:37 -07:00
Justine Tunney 0c70e8963d Add notpossible keyword
This is the same as `unreachable` except it always traps violations,
even if we're not running in MODE=dbg. This is useful for impossible
conditions relating to system calls. It avoids terrifying bugs where
control falls through to an unrelated function.
2022-09-03 20:35:31 -07:00
Justine Tunney b66bd064d8 Improve quality of uname/gethostname/getdomainname 2022-09-03 20:20:40 -07:00
Justine Tunney c5c4dfcd21 Improve quality of raise(), abort(), and tkill()
This change fixes a nasty bug where SIG_IGN and SIG_DFL weren't working
as advertised on BSDs. This change also fixes the tkill() definition on
MacOS so it maps to __pthread_kill().
2022-09-03 20:17:54 -07:00
Justine Tunney c5659b93f8 Create unit test framework for subprocesses 2022-09-03 18:02:01 -07:00
Gavin Hayes 263711965f Change sigaction_f to match sysv signature (#585) 2022-09-02 05:08:35 -07:00
Paul Kulchenko 33b5b5b312 Remove path escaping from LaunchBrowser (#584)
This is so you can have `?query=args`.
2022-08-31 22:14:06 -07:00
Gavin Hayes d213a48190 Implement inet_pton for AF_INET6 (#583) 2022-08-31 15:14:42 -07:00
Paul Kulchenko 3f0414e1cb Fix JSON single quote escaping (fixes #512) 2022-08-31 07:53:57 -05:00
Gavin Hayes adaaeb7dd0 Add itimerval to sys/time.h (#572) 2022-08-26 09:44:41 -07:00
Paul Kulchenko 164e2ab27b Rename MapContentType to ProgramContentType (#570) 2022-08-24 09:35:17 -07:00
Paul Kulchenko d2cec808dc Fix redbean to check for file read result when -F option is used (#569) 2022-08-23 20:54:51 -07:00
Paul Kulchenko 91a3252434 Add MapContentType to redbean (#566) 2022-08-22 22:02:58 -07:00
Justine Tunney 8cab92a069 Release awk-2022-08-22.com 2022-08-22 21:28:19 -07:00
Justine Tunney 5a632cf72d Add lchown, lchmod, statvfs, fstatvfs 2022-08-22 20:50:16 -07:00
Gautham 7b74f6e20e Tidy up pydoc help (#567) 2022-08-22 20:40:00 -07:00
Justine Tunney 253cc1754b Shave another 2% off GNU Make latency 2022-08-22 17:42:32 -07:00
Justine Tunney 8722006c46 Shave another 10% off GNU Make latency
We avoid memcpy() by creating a likely branch with a constant size.
2022-08-22 17:04:54 -07:00
Justine Tunney 58d22c7c3d Handle missing stdio handles on Windows 2022-08-22 17:03:11 -07:00
Justine Tunney 2227cefa5d Run fix-third-party.py and man2txt on sed 2022-08-21 22:26:48 -07:00
Justine Tunney f0bf9f3389 Import NetBSD sed 2022-08-21 22:26:48 -07:00
Paul Kulchenko 04cd62c319 Update redbean GetResponseBody to take response compression into account (#561) 2022-08-21 22:26:41 -07:00
Paul Kulchenko e204fc0820 Update redbean If-Modified-Since check to accept same timestamp (#562) 2022-08-21 22:03:30 -07:00
Justine Tunney b76540105d Release Cosmopolitan v2.0.1 2022-08-21 18:59:20 -07:00
Justine Tunney 14e3025749 Release redbean 2.0.17 2022-08-21 18:35:53 -07:00
Justine Tunney 51fa5535ac Fix regression in Fetch() 2022-08-21 18:32:57 -07:00
Justine Tunney bea0b55576 Fix If-Modified-Since regression 2022-08-21 18:05:05 -07:00
Justine Tunney 57b39d8793 Revert recent fixupobj.com change 2022-08-21 17:33:54 -07:00
Justine Tunney d52170ca34 Avoid zip -d redbean.com foo issue on Windows 2022-08-21 16:32:57 -07:00
Justine Tunney bf757c4646 Fix _Thread_local with fork() on Windows 2022-08-21 15:51:44 -07:00
Justine Tunney 99a92048b4 Get awk to build and make it hackable 2022-08-21 13:39:20 -07:00
Justine Tunney 2f1679e5cf Import The One True Awk
https://github.com/onetrueawk/awk
2022-08-21 13:38:12 -07:00
Justine Tunney ee49b71be2 Fix build 2022-08-21 01:11:14 -07:00
Justine Tunney 73845be1f0 Restore zip.com and .symtab files
This change restores the .symtab symbol table files in our flagship
programs (e.g. redbean.com, python.com) needed to show backtraces. This
also rolls back earlier changes to zip.com w.r.t. temp directories since
the right way to do it turned out to be the -b DIR flag.

This change also improves the performance of zip.com. It turned out
mmap() wasn't being used, because zip.com was assuming a 4096-byte
granularity, but cosmo requires 65536. There was also a chance to speed
up stdio scanning using the unlocked functions.
2022-08-21 00:17:20 -07:00
Justine Tunney 35203c0551 Do some string library work 2022-08-20 22:17:14 -07:00
Justine Tunney 83d41e4588 Clean up some code 2022-08-20 12:32:51 -07:00
Paul Kulchenko a51edaa0db Remove .heartbeat.lua from redbean (#554)
It's been replaced by `OnServerHeartbeat`.
2022-08-20 01:31:58 -07:00
Justine Tunney d76dfadc7a Reduce GNU Make latency 17% for cosmo
fgets() is now 4x faster which makes Make 2% faster. Landlock Make now
has a builtin $(uniq ...) function that uses critbit trees rather than
functional programming. Since uniq is the most important function this
optimization makes our cold start latency 15% faster.
2022-08-19 15:29:24 -07:00
Justine Tunney 8835b82a7c Avoid sandboxing directory prerequisites
Landlock Make will no longer sandbox prerequisites that end with a
trailing slash. This means you can use use directory prerequisites
for detecting deleted files when using using globbing, without the
effect of unveiling the entire directory. When you do want make to
unveil directories, you can omit the trailing slash.
2022-08-19 13:02:25 -07:00
Daniil Kulchenko 2827df688a Fix M1 Rosetta signal handling (#455) (#553)
Rosetta does something strange to the signal handling registers but
setting SA_SIGINFO prevents the issue from happening. Set the flag
on XNU to work around the issue.
2022-08-19 13:00:50 -07:00
Justine Tunney 1ce101c5a5 Release Cosmopolitan 2.0 2022-08-18 18:22:16 -07:00
Justine Tunney fcbf64b906 Remove warnings from amalgamation header 2022-08-18 18:16:53 -07:00
Justine Tunney 283fd9e590 Pledge tty on Python ioctl test 2022-08-18 18:02:49 -07:00
Justine Tunney 6bb9ce553e Remove some includes 2022-08-18 17:46:34 -07:00
Justine Tunney 75832f7379 Polyfill sysinfo() a ilttle bit on BSDs 2022-08-18 15:53:11 -07:00
Gavin Hayes 897e33ccc4 Fix stdio fmt of "%.0e" and "%.0g" (#544)
* Fix %.0e by always rounding even if precision is 0
* Fix %.0g by treating it the same as %.1g
* Fix %g tests to match glibc, add more tests
2022-08-18 15:53:07 -07:00
Gavin Hayes 27416e7dd6 Add fake support for locale="" (#546) 2022-08-18 15:51:23 -07:00
Gavin Hayes 6c3048821c Add libc/str/nltypes.h to langinfo.h (#549) 2022-08-18 15:45:56 -07:00
Harmen Stoppels 2d5f2bebb0 Remove unused variable (#536) 2022-08-18 15:42:48 -07:00
Jared Miller db509ee158 LaunchBrowser by default in redbean-demo.com (#547) 2022-08-18 12:01:44 -07:00
Gavin Hayes 640b9bfb48 Add test/libc/mem/putenv_test.c (#545) 2022-08-18 00:11:35 -07:00
Justine Tunney 30e1c5bca9 Add statfs() and fstatfs() to redbean unix module 2022-08-17 23:27:17 -07:00
Justine Tunney 2d479f7b11 Update definitions with latest help.txt changes 2022-08-17 21:57:08 -07:00
Justine Tunney d7372e651d Fix arg redbean lua definition 2022-08-17 21:23:44 -07:00
Justine Tunney 8e176fb026 Reduce build graph by another 14 percent
That's a 37% reduction total from what it was on 2022-05-23.
2022-08-17 20:01:21 -07:00
Justine Tunney c2211c9e63 Polyfill statfs() and fstatfs() on Windows 2022-08-17 19:01:51 -07:00
Justine Tunney f7ee9d7d99 Polyfill statfs() and fstatfs() on BSD distros 2022-08-17 14:54:03 -07:00
Gavin Hayes e3fe127ccd Change sys_lseek return value from i32 to i64 (#539) 2022-08-17 14:21:24 -07:00
Danny Robinson fbdab72e3b Create definitions.lua (#540) 2022-08-17 14:20:07 -07:00
Gavin Hayes 1f735a4af3 Prevent NPE after clearenv (#542) 2022-08-17 14:15:23 -07:00
Justine Tunney e5f705ace3 Fix redbean doc errors reported by stellartux 2022-08-17 00:10:17 -07:00
Justine Tunney a1aaf23dc1 Add more apis to redbean unix module
- Document unix.fcntl()
- Add POSIX Advisory Locks
- Add mask parameter to unix.poll()
- Add lowest parameter to unix.dup()
2022-08-16 23:23:34 -07:00
Justine Tunney ce588dd56b Release pledge.com 1.7 and landlockmake.com 1.3
- pledge("chown") now supported
- pledge("stdio") now allows killing self
- Write tests for pselect() and ppoll()
2022-08-15 19:52:00 -07:00
Justine Tunney 255d834f8f Put ape.macho back in build/bootstrap 2022-08-15 15:45:13 -07:00
Justine Tunney f0701d2a24 Make improvements
- Polyfill pselect() on Windows
- Add -O NOFILE flag to pledge.com
- Polyfill ppoll() on NetBSD, XNU, and Windows
- Support negative numbers and errno in sizetol()
- Add .RSS, .NOFILE, and .MAXCORE to Landlock Make
- Fix issue with .PLEDGE preventing touching of output files
- Add __watch() function (like ftrace) for logging memory changes
2022-08-15 15:20:36 -07:00
Justine Tunney d3b599a796 Remove ANSI WIN32 functions and undocumented ADVAPI32 2022-08-15 09:01:40 -07:00
Justine Tunney 0cf9716039 Improve build latency of repository 2022-08-14 22:19:46 -07:00
Justine Tunney 7ab15e0b23 Add .PLEDGE/.CPU/.MEMORY/etc. to Landlock Make 1.2 2022-08-14 20:16:44 -07:00
Justine Tunney 6c0bbfac4a Fix some bugs 2022-08-14 13:28:07 -07:00
tkchia 5584f6adcf Improve detection of boot media on bare metal (#535)
hello.com now starts up correctly on QEMU when run as a hard
disk image.
2022-08-14 08:32:58 -07:00
Justine Tunney d36d0634db Add automatic TMPDIR setup/teardown to GNU Make
We now guarantee TMPDIR will be defined on a per build rule basis. It'll
be an absolute path. It'll be secure and unique. It'll be rm -rf'd after
the last shell script line in your build rule is executed. If $TMPDIR is
already defined, then it'll be created as a subdirectory of your $TMPDIR
and then replace the variable with the new definition. The Landlock Make
repository will be updated with examples shortly after this change which
shall be known as Landlock Make 1.1.1.

See #530
2022-08-14 02:03:41 -07:00
Justine Tunney e1699c5b68 Fix issue with stack creation on OpenBSD 2022-08-14 01:19:58 -07:00
Justine Tunney 2e54c8f9fc Change default build mode to use -O2 not -Og 2022-08-14 01:18:15 -07:00
Justine Tunney ead3fc2b31 Fix Landlock Make so it can read pattern rule vars
It turned out that specifying all SRCS and INCS as dependencies on the
pattern rules for all headers, caused `make` memory usage to skyrocket
from 40mb ot 160mb. This change also reduces the build graph another 4%.
2022-08-13 17:23:05 -07:00
tkchia 62ca1b0902 Get hello.com working on metal again (#529)
* Fix deterministic startup stack setup, especially for bare metal
* Implement __enable_tls() on bare metal
* Get __get_tls_privileged() working on bare metal
2022-08-13 16:14:02 -07:00
Justine Tunney ad775a75b8 Restore strict header checking 2022-08-13 16:02:01 -07:00
Justine Tunney b212f49ef4 Fix build rule for pushing cosmo nightlies 2022-08-13 15:16:06 -07:00
Justine Tunney e62d7b8789 Fix return type on lock macros
Fixes #515
2022-08-13 14:18:02 -07:00
Justine Tunney 0ea0d33a77 Reduce build graph by another eight percent 2022-08-13 13:11:56 -07:00
Justine Tunney 367d06d9e4 Fold LIBC_UNICODE into LIBC_STR 2022-08-13 08:42:32 -07:00
Justine Tunney 17aea99bb3 Fold LIBC_ALG into LIBC_MEM 2022-08-13 08:32:34 -07:00
Justine Tunney 7cf66bc161 Prevent Make from talking to public Internet
This change introduces the nointernet() function which may be called to
prevent a process and its descendants from communicating with publicly
routable Internet addresses. GNU Make has been modified to always call
this function. In the future Landlock Make will have a way to whitelist
subnets to override this behavior, or disable it entirely. Support is
available for Linux only. Our firewall does not require root access.

Calling nointernet() will return control to the caller inside a new
process that has a SECCOMP BPF filter installed, which traps network
related system calls. Your original process then becomes a permanent
ptrace() supervisor that monitors all processes and threads descending
from the returned child. Whenever a networking system call happens the
kernel will stop the process and wakes up the monitor, which then peeks
into the child memory to read the sockaddr_in to determine if it's ok.

The downside to doing this is that there can be only one supervisor at a
time using ptrace() on a process. So this firewall won't be enabled if
you run make under strace or inside gdb. It also makes testing tricky.
2022-08-12 21:51:39 -07:00
Justine Tunney 8a0a2c0c36 Fold LIBC_RAND into LIBC_STDIO/TINYMATH/INTRIN 2022-08-11 12:32:00 -07:00
Justine Tunney 05b8f82371 Fold LIBC_BITS into LIBC_INTRIN 2022-08-11 12:13:18 -07:00
Justine Tunney 625aa365f1 Refactor pledge() to be more configurable
The earlier iterations did too much guesswork when it came to things
like stderr logging and syscall origin verification. This change will
make things more conformant to existing practices. The __pledge_mode
extension now can be configured in a better way.

There's also a new `-q` flag added to pledge.com, e.g.

    o//tool/build/pledge.com -qv. ls

Is a good way to disable warnings about `tty` access attempts.
2022-08-11 11:35:30 -07:00
Justine Tunney 6b8b58fdf5 Update build/bootstrap/make.com 2022-08-11 00:37:59 -07:00
Justine Tunney 10fd8bdb70 Unbloat the build
This change resurrects ae5d06dc53
2022-08-11 00:15:29 -07:00
Justine Tunney 2d64b9994b Avoid creating temporary output files
This change also removes the futimens() call on the Landlock Make output
file workaround, since it caused problems with commands like fixupobj
which modify-in-place. It turns out if a file is opened for writing and
then no writes actually occur, then the modified time doesn't change.
2022-08-10 22:31:16 -07:00
Justine Tunney 4c3ab6d11e Make it easier to configure Landlock Make 2022-08-10 21:24:31 -07:00
Justine Tunney 7b993d561c Use private keyword on makefile target variables 2022-08-10 18:36:11 -07:00
Justine Tunney e015defa70 Fix Make regression with .UNSANDBOXED 2022-08-10 17:06:59 -07:00
Justine Tunney 37ebd8f8d5 Don't generate output file for .PHONY targets 2022-08-10 15:52:06 -07:00
Justine Tunney 7e1c78b4f3 Cherry-pick bug fixes and Landlock Make 1.0.2 2022-08-10 12:56:45 -07:00
Justine Tunney c1d99676c4 Revert "Unbloat build config"
This reverts commit ae5d06dc53.
2022-08-10 12:44:56 -07:00
Justine Tunney 95f54eeb40 Release Landlock Make v1.0.1 2022-08-10 12:17:06 -07:00
Justine Tunney 5c5cf0e01d Revert "Release Landlock Make 1.0"
This reverts commit 6746d768c6.
2022-08-10 12:13:47 -07:00
Gavin Hayes 3b81bec5ae Prevent utimesat / zipos breakage (#532) 2022-08-10 10:41:09 -07:00
Danny Robinson de48bae30e Improve cosmopolitan libc documentation (#516) 2022-08-10 10:21:35 -07:00
Justine Tunney 6746d768c6 Release Landlock Make 1.0 2022-08-10 09:27:04 -07:00
Justine Tunney 4e939d1761 Release pledge.com v1.6 2022-08-10 09:21:51 -07:00
Justine Tunney ae5d06dc53 Unbloat build config
- 10.5% reduction of o//depend dependency graph
- 8.8% reduction in latency of make command
- Fix issue with temporary file cleanup

There's a new -w option in compile.com that turns off the recent
Landlock output path workaround for "good commands" which do not
unlink() the output file like GNU tooling does.

Our new GNU Make unveil sandboxing appears to have zero overhead
in the grand scheme of things. Full builds are pretty fast since
the only thing that's actually slowed us down is probably libcxx

    make -j16 MODE=rel
    RL: took 85,732,063µs wall time
    RL: ballooned to 323,612kb in size
    RL: needed 828,560,521µs cpu (11% kernel)
    RL: caused 39,080,670 page faults (99% memcpy)
    RL: 350,073 context switches (72% consensual)
    RL: performed 0 reads and 11,494,960 write i/o operations

pledge() and unveil() no longer consider ENOSYS to be an error.
These functions have also been added to Python's cosmo module.

This change also removes some WIN32 APIs and System Five magnums
which we're not using and it's doubtful anyone else would be too
2022-08-10 04:43:09 -07:00
Justine Tunney 133c693650 Work around Landlock output inode in compile.com
This change fixes Landlock Make so that only the output target file is
unveiled, rather than unveiling the directory that contains it. This
gives us a much stronger sandbox. It also helped identify problematic
build code in our repo that should have been using o/tmp instead.

Landlock isn't able to let us unveil files that don't exist. Even if
they do, then once a file is deleted, the sandboxing for it goes away.
This caused problems for Landlock Make because tools like GNU LD will
repeatedly delete and recreate the output file. This change uses the
compile.com wrapper to ensure on changes happen to the output inode.

New binary available on https://justine.lol/make/

Fixes #528
2022-08-09 07:55:44 -07:00
Justine Tunney c464f45692 Clean up some variables 2022-08-09 02:55:24 -07:00
Justine Tunney d4744048ba Release pledge.com v1.5 2022-08-08 21:27:44 -07:00
Justine Tunney 6b3d257588 Use LD_PRELOAD to inject pledge() in glibc progs
We're now able to drop both `exec` and `prot_exec` privileges
automatically when launching glibc dynamic executables. We also have
really outstanding standard error logging now, that explains which
promises are needed, even in cases where `exec` is used.
2022-08-08 21:27:02 -07:00
Justine Tunney 0277d7d6e9 Rewrite Linux pledge() code so it can be a payload
It's now possible to build our pledge() polyfill as a dynamic shared
object that can be injected into a glibc executable using LD_PRELOAD
2022-08-08 11:41:08 -07:00
Justine Tunney 7bd4179b9b Add restart_syscall to pledge() 2022-08-07 22:58:33 -07:00
Justine Tunney b77cae2d57 Fix some regressions with execution 2022-08-07 22:10:18 -07:00
Justine Tunney 5546559034 Improve pledge() usability and consistency
- We now kill the program on violations like OpenBSD
- We now print a message explaining which promise is needed
- This change also fixes a linkage bug with thread local storage
- Your sigaction() handlers should now be more thread safe

A new `__pledge_mode` global has been introduced to make pledge() more
customizable on Linux. For example:

    __attribute__((__constructor__)) static void init(void) {
      __pledge_mode = SECCOMP_RET_ERRNO | EPERM;
    }

Can be used to restore our old permissive pledge() behavior.
2022-08-07 16:18:33 -07:00
Justine Tunney 13c1c45075 Make some last minute improvements to make.com 2022-08-07 05:59:53 -07:00
Justine Tunney 6a5717a48f Make more libc improvements
- Make memmem() faster
- Make readdir() thread safe
- Remove 64kb limit from mkdeps.com
- Add old crypt() function from Musl
- Improve new fix-third-party.py tool
- Improve libc/isystem/ headers and fix bugs
2022-08-06 17:18:40 -07:00
Justine Tunney a8cf0f7e89 Add more missing libc functionality 2022-08-06 10:50:51 -07:00
Justine Tunney cf93ecbbb2 Prove that Makefile is fully defined
The whole repository is now buildable with GNU Make Landlock sandboxing.
This proves that no Makefile targets exist which touch files other than
their declared prerequisites. In order to do this, we had to:

  1. Stop code morphing GCC output in package.com and instead run a
     newly introduced FIXUPOBJ.COM command after GCC invocations.

  2. Disable all the crumby Python unit tests that do things like create
     files in the current directory, or rename() files between folders.
     This ended up being a lot of tests, but most of them are still ok.

  3. Introduce an .UNSANDBOXED variable to GNU Make to disable Landlock.
     We currently only do this for things like `make tags`.

  4. This change deletes some GNU Make code that was preventing the
     execve() optimization from working. This means it should no longer
     be necessary in most cases for command invocations to be indirected
     through the cocmd interpreter.

  5. Missing dependencies had to be declared in certain places, in cases
     where they couldn't be automatically determined by MKDEPS.COM

  6. The libcxx header situation has finally been tamed. One of the
     things that makes this difficult is MKDEPS.COM only wants to
     consider the first 64kb of a file, in order to go fast. But libcxx
     likes to have #include lines buried after huge documentation.

  7. An .UNVEIL variable has been introduced to GNU Make just in case
     we ever wish to explicitly specify additional things that need to
     be whitelisted which aren't strictly prerequisites. This works in
     a manner similar to the recently introduced .EXTRA_PREREQS feature.

There's now a new build/bootstrap/make.com prebuilt binary available. It
should no longer be possible to write invalid Makefile code.
2022-08-06 04:05:08 -07:00
Justine Tunney acdf591833 Avoid long double timestamps in redbean 2022-08-05 19:24:05 -07:00
Justine Tunney 8d9ac3da50 Improve redbean memory locality
Doing this improves message latency and throughput, since we're
consolidating the 504 bytes of static memory which has the most
churn into a single place.
2022-08-05 17:34:53 -07:00
Justine Tunney 638c56e3a5 Propagate nil in Lua APIs more often 2022-08-05 17:34:13 -07:00
Justine Tunney c9d7838213 Enter meltdown mode if ProgramMaxWorkers triggers 2022-08-05 15:42:17 -07:00
Paul Kulchenko 9c06067c84 Add max worker processing to redbean (#520)
* Add max worker processing
* Introduce ProgramMaxWorkers() API
2022-08-05 15:21:27 -07:00
Justine Tunney 9d7514a906 Release redbean 2.0.16 2022-08-05 15:05:57 -07:00
Justine Tunney 84e89a55d4 Fix redbean -A asset storing flag 2022-08-05 15:04:13 -07:00
Justine Tunney db8217b37b Benchmark signal delivery 2022-08-05 15:01:23 -07:00
Paul Kulchenko 449fb2fb59 Fix stderr being improperly closed during daemonization (#513)
This may happen when ProgramLogPath is used after a file descriptor is
already used by an earlier call (for example, to open an SQLite file).
2022-08-05 14:57:17 -07:00
Jules Bertholet 5f552338d6 Change magnum types to match other platforms (#525) 2022-08-05 14:50:08 -07:00
Justine Tunney af3df0893b Refactor gettimeofday() 2022-08-05 02:13:41 -07:00
Justine Tunney 4238e4def9 Change type of errnos to errno_t
https://github.com/jart/cosmopolitan/discussions/521
2022-08-05 02:13:10 -07:00
Theta Nil ab301401c7 Add gensvg demo to redbean (#524)
gensvg demo and mk for it
2022-08-03 05:21:41 -07:00
Paul Kulchenko 42bd79a461 Add OnServerHeartbeat to Redbean (#522) 2022-08-02 20:41:44 -07:00
Justine Tunney 066ed2b2b2 Release pledge.com v1.4 2022-07-25 00:02:42 -07:00
Justine Tunney bd118dafa0 Don't retrieve uid/gid from auxv 2022-07-24 23:54:07 -07:00
Justine Tunney e98514cdb7 Plug a hole in pledge()
Günther Noack points out that O_RDONLY|O_TRUNC will modify a file.
2022-07-24 23:41:59 -07:00
Justine Tunney 3443039f34 Perform code cleanup on test pledges 2022-07-24 22:34:13 -07:00
Justine Tunney 3d2cf95af1 Remove malloc() dependency on pledge() / unveil()
This change also fixes a bug with gettid() being incorrect after fork().
We now implement the ENOENT behavior for getauxval(). The getuid() etc.
system calls are now faster too. Plus issetugid() will work on BSDs.
2022-07-24 21:51:37 -07:00
Justine Tunney c921dc78f0 Release pledge.com v1.3 2022-07-24 08:35:13 -07:00
Justine Tunney 6c71bd5969 Further improve unveil() implementation
This change addresses review comments from Günther Noack on GitHub.
We're now blacklisting truncate() and setxattr() since Landlock lets
them operate on veiled files. The restriction has been lifted on using
unveil() multiple times, since Landlock does that well.
2022-07-24 06:39:25 -07:00
Justine Tunney 8593580d0a Fix lsqlite3.lversion() and document local changes
Fixes #507
2022-07-24 05:13:28 -07:00
Justine Tunney f968e2a726 Improve pledge() and unveil() further
- Fix getpriority()
- Add AT_MINSIGSTKSZ
- Fix bugs in BPF code
- Show more stuff in printargs.com
- Write manual test for pledge.com
- pledge() now generates tinier BPF code
- Have pledge("exec") only enable execve()
- Fix pledge.com chroot setuid functionality
- Improve pledge.com unveiling of ape loader
2022-07-24 03:10:16 -07:00
Jared Miller 31ac58a57b Adds RFC driven BASE64, MD5, and SHA tests (#504) 2022-07-23 19:01:32 -07:00
Paul Kulchenko 638e14bbf3 Improve Redbean shutdown (#506)
* Update redbean shutdown to call OnServerStop when all shutdown/logging is done

* Move closing file descriptors during daemonization earlier

This should fix using opened file descriptors, for example, SQLite DB
files and redbean itself when StoreAsset is used. Fixes #182.

* Move opening logs earlier to capture logs from Listen and .init.lua

* Move pidpath handling outside of daemonize, as it can be used independently
2022-07-23 18:58:31 -07:00
Justine Tunney 03dd14c298 Fix MODE=rel build 2022-07-23 12:10:08 -07:00
Justine Tunney 16fc83f9ce Explicitly disable Linux capabilities 2022-07-23 12:06:41 -07:00
Justine Tunney ffedbfe14d Fix copy/paste error in redbean docs 2022-07-23 07:37:39 -07:00
Justine Tunney 3828c08aa0 Release redbean 2.0.15 2022-07-23 07:22:19 -07:00
Justine Tunney ce5cb8a2f8 Have redbean Lua repl show utf-8 when it's valid
Otherwise EncodeLua() will assume strings are binary and display them
using hex sequences.
2022-07-23 07:22:19 -07:00
Justine Tunney 31e4b0867b Pledge and unveil redbean's unit tests 2022-07-23 07:22:19 -07:00
Arham Jain 513a659dd2 Add ## to ioctl VA_ARGS (#503) 2022-07-22 21:51:12 -07:00
Justine Tunney 48ce3ad7cc Do some work on redbean
- Rewrite Slurp() API to be like string.sub()
- Introduce a new Barf() API for creating files
- Update Redbean `-S` sandbox flag to do unveiling
2022-07-22 20:44:24 -07:00
Justine Tunney 742251dd92 Release redbean 2.0.14 2022-07-22 17:07:25 -07:00
Justine Tunney 76d2f68c91 Release pledge.com v1.1
This change fixes bugs, adds more system calls, and improves
compatibility with OpenBSD. Going forward, versions on the web will be
pinned to a permanent version. There were many other changes over the
last week which also improved this new release.
2022-07-22 13:44:00 -07:00
Paul Kulchenko b5904947e9 Add GetResponseBody to redbean (#502)
* Add GetResponseBody to redbean to get access to generated response
* Update GetStatus to return status code set by redbean itself (outside
  of Lua code)
2022-07-22 11:22:39 -07:00
Justine Tunney 516b68606f Add pretty printing to redbean serializers 2022-07-22 10:10:33 -07:00
Justine Tunney 6bb8b26d70 Fix NPE in new unbuffer command 2022-07-22 07:31:42 -07:00
Justine Tunney 31e746c937 Add more missing libc/libc++ functions
- Add sched_rr_get_interval()
- Add `unbuffer` command example
- Add more locale function stubs
- Vendor most of remaining libcxx content
2022-07-22 07:20:21 -07:00
Justine Tunney 5a2bb07b36 Add ctermid and L_ctermid 2022-07-22 04:39:04 -07:00
Justine Tunney 84caee23ba Make sorted serialization faster
Redbean Lua and JSON serialization now goes faster because we're now
inserting object entries into tree data structure rather than making
an array and sorting it at the end. For example, when serializing an
object with 10,000 entries this goes twice as fast. However it still
goes slower than saying EncodeJson(x, {sorted=false}).
2022-07-22 04:19:01 -07:00
Jared Miller 9de3d8f1e6 Revert whitespace fixes to third_party (#501) 2022-07-21 21:46:07 -07:00
Jared Miller d4000bb8f7 Correct more typos (#500) 2022-07-21 20:53:30 -07:00
Justine Tunney 8b469389f6 Remove plenty of makefile misconfigurations 2022-07-21 09:20:59 -07:00
Justine Tunney 9172fd42a0 Add Landlock sandboxing to GNU Make 2022-07-21 09:16:27 -07:00
Justine Tunney 1837dc2e85 Make improvements
- Introduce path module to redbean
- Fix glitch with linenoise printing extra line on eof
- Introduce closefrom() and close_range() system calls
- Make file descriptor closing more secure in pledge.com
2022-07-21 03:36:42 -07:00
Paul Kulchenko 439ad21b12 Allow .well-known/ hidden paths (#499) 2022-07-20 23:26:49 -07:00
Jared Miller 7e2eae5c15 Remove trailing whitespace from all files (#497) 2022-07-20 20:31:16 -07:00
Justine Tunney d3f3cb7ab4 Fix build 2022-07-20 14:00:38 -07:00
Justine Tunney 097634d75c Make pledge.com binary 4kb smaller 2022-07-20 13:59:28 -07:00
jared ed205e98a1 WIP: Correct all typos (#498) 2022-07-20 14:01:15 -07:00
Justine Tunney 98254a7c1f Make pledge() and unveil() work amazingly
This change reconciles our pledge() implementation with the OpenBSD
kernel source code. We now a polyfill that's much closer to OpenBSD's
behavior. For example, it was discovered that "stdio" permits threads.
There were a bunch of Linux system calls that needed to be added, like
sched_yield(). The exec / execnative category division is now dropped.
We're instead using OpenBSD's "prot_exec" promise for launching APE
binaries and dynamic shared objects. We also now filter clone() flags.

The pledge.com command has been greatly improved. It now does unveiling
by default when Landlock is available. It's now smart enough to unveil a
superset of paths that OpenBSD automatically unveils with pledge(), such
as /etc/localtime. pledge.com also now checks if the executable being
launched is a dynamic shared object, in which case it unveils libraries.

These changes now make it possible to pledge curl on ubuntu 20.04 glibc:

    pledge.com -p 'stdio rpath prot_exec inet dns tty sendfd recvfd' \
        curl -s https://justine.lol/hello.txt

Here's what pledging curl on Alpine 3.16 with Musl Libc looks like:

    pledge.com -p 'stdio rpath prot_exec dns inet' \
        curl -s https://justine.lol/hello.txt

Here's what pledging curl.com w/ ape loader looks like:

    pledge.com -p 'stdio rpath prot_exec dns inet' \
        o//examples/curl.com https://justine.lol/hello.txt

The most secure sandbox, is curl.com converted to static ELF:

    o//tool/build/assimilate.com o//examples/curl.com
    pledge.com -p 'stdio rpath dns inet' \
        o//examples/curl.com https://justine.lol/hello.txt

A weird corner case needed to be handled when resolving symbolic links
during the unveiling process, that's arguably a Landlock bug. It's not
surprising since Musl and Glibc are also inconsistent here too.
2022-07-19 21:33:49 -07:00
Justine Tunney 92cb144fff Revise wording of support vector in readme 2022-07-19 16:05:19 -07:00
Justine Tunney f716435260 Whitelist some more system calls with pledge() 2022-07-19 02:54:10 -07:00
Justine Tunney 6e52a84a51 Prevent unveil() from being used multiple times
See #493
2022-07-19 02:12:32 -07:00
Justine Tunney f7e22a33a1 Disable kernel verbose strace output 2022-07-19 00:33:36 -07:00
Justine Tunney 69f4152f38 Always initialize thread local storage
We had previously not enabled TLS in MODE=tiny in order to keep the
smallest example programs (e.g. life.com) just 16kb in size. But it
was error prone doing that, so now we just always enable it because
this change uses hacks to ensure it won't increase life.com's size.

This change also fixes a bug on NetBSD, where signal handlers would
break thread local storage if SA_SIGINFO was being used. This looks
like it might be a bug in NetBSD, but it's got a simple workaround.
2022-07-19 00:21:46 -07:00
Justine Tunney 057e8f5b54 Fix O_CLOEXEC behavior with unveil() 2022-07-18 21:05:46 -07:00
Justine Tunney bf59defc0c Fix GitHub Actions build
This is an unusual failure that seems to happen intermittently across
the various build modes. It should not be possible for life.elf to be
exiting with status zero.
2022-07-18 20:45:18 -07:00
Paul Kulchenko 574eba8352 Add redbean OnLogLatency hook (#495) 2022-07-18 20:17:14 -07:00
Stephen Gregoratto 6598940d8a Make unveil() improvements (#493)
- Merge the multiple masks to just one.
- Add documentation to for sys_unveil.
- Inline the chomp function in the unveil tool.
2022-07-18 08:26:40 -07:00
Justine Tunney e81edf7b04 Improve pledge() and unveil()
The pledge.com command now supports the new [WIP] unveil() support. For
example, to strongly sandbox our command for listing directories.

    o//tool/build/assimilate.com o//examples/ls.com
    pledge.com -v /etc -p 'stdio rpath' o//examples/ls.com /etc

This file system sandboxing is going to be perfect for us, because APE
binaries are self-contained static executables that really don't use the
filesystem that much. On the other hand, with non-static executables,
sandboxing is going to be more difficult. For example, here's how to
sandbox the `ls` command on the latest Alpine:

    pledge.com -v rx:/lib -v /usr/lib -v /etc -p 'stdio rpath exec' ls /etc

This change fixes the `execpromises` API with pledge().

This change also adds unix.unveil() to redbean.

Fixes #494
2022-07-18 07:58:20 -07:00
Justine Tunney b1d9d11be1 Simplify TLS and reduce startup latency
This change simplifies the thread-local storage support code. On Windows
and Mac OS X the startup latency of __enable_tls() has been reduced from
30ms to 1ms. On Windows, TLS memory accesses will now go much faster due
to better self-modifying code that prevents a function call and acquires
our thread information block pointer in a single instruction.
2022-07-18 04:10:54 -07:00
Justine Tunney 38c3fa63fe Write some tests for unveil()
See #490
2022-07-18 02:27:26 -07:00
Stephen Gregoratto 1c6b5c0acd [WIP] Polyfill OpenBSD unveil for Linux (#490) 2022-07-18 02:12:42 -07:00
Justine Tunney 4f4889ddf7 Use futexes on OpenBSD and improve threading 2022-07-17 19:59:49 -07:00
Justine Tunney 5b11033d4d Add redbean -I flag for launching browser
It's now possible with any redbean (including redbean-original) to
launch the system web browser without having to use the Lua API. For
example, you can create an args file:

    echo -I/ >.args
    zip redbean-original.com .args

That will white-label redbean so it launches a specific page when you
double-click on the executable.

See https://github.com/jart/cosmopolitan/discussions/472
2022-07-17 06:12:57 -07:00
Justine Tunney 4d25f8c3c9 Add tcp syn packet fingerprinting to redbean
This change also fixes bugs in enoprotoopt reporting with setsockopt and
getsockopt error returns.
2022-07-17 02:43:49 -07:00
Justine Tunney 866b21a151 Get redbean -X running in blinkenlights again
This change improves the loading of APE executables in Blinkenlights and
adds some system call wrappers that were previous missing.
2022-07-16 11:52:45 -07:00
Wiebe 4700984456 Update Redbean help.txt (#492)
Fix typos and add previously undocumented functions:

- GetHttpReason
- IsHiddenPath
- IsAcceptablePath
- IsReasonablePath
- ProgramTimeout
2022-07-16 11:00:28 -07:00
S. Neuhaus ea04d00752 Fix typo in readme (#486) 2022-07-16 10:56:10 -07:00
Justine Tunney 6c724c0f1a Update experiment with tty audio 2022-07-15 23:07:32 -07:00
Justine Tunney aa34340f3d Add pipelining to cocmd 2022-07-15 20:47:20 -07:00
Justine Tunney b4e38851ff Fix pthread isystem reference
Fixes #489
2022-07-15 18:48:24 -07:00
Justine Tunney 13d67fed38 Add latest Linux syscalls including landlock
See #485
2022-07-15 18:29:49 -07:00
Justine Tunney 2e3958c6dc Make some touchups 2022-07-15 18:07:34 -07:00
Justine Tunney 6c49e36537 Release redbean 2.0.13 2022-07-15 07:47:36 -07:00
Justine Tunney 7f966de489 Limit pledge.com default virtual mem to total ram 2022-07-15 06:53:01 -07:00
Justine Tunney baf51a4a23 Add utf-8 validation to ljson 2022-07-15 06:20:07 -07:00
Justine Tunney ccd057a85d Permit MAP_POPULATE with pledge() 2022-07-14 13:04:28 -07:00
Justine Tunney 0a589add41 Third time's a charm with stack remaining checker 2022-07-14 07:38:44 -07:00
Justine Tunney 28b9d9f781 Fix C stack remaining check in MODE=tiny 2022-07-14 07:23:15 -07:00
Justine Tunney bcdd5c5f74 Fix ljson stack check 2022-07-14 07:07:17 -07:00
Justine Tunney 4901e1b8cb Fix MODE=tiny build 2022-07-14 05:42:24 -07:00
Justine Tunney 1d744ea11b Improve upon the new pledge command 2022-07-14 04:39:45 -07:00
Justine Tunney b707fca77a Make JSON parser perfectly conformant 2022-07-13 23:02:19 -07:00
Justine Tunney 60164a7266 Add assimilate.com command for APE binaries 2022-07-13 20:56:24 -07:00
Gautham 0cea6c560f Make JSON parser nearly perfectly compliant (#483) 2022-07-13 07:38:23 -07:00
Justine Tunney 1d490fcb94 Add pledge.com for launching commands in a sandbox 2022-07-13 04:31:46 -07:00
Gautham 12d9f7ade6 Make ljson more strict (#482) 2022-07-13 02:39:19 -07:00
Justine Tunney 30cc2c8dc1 Release redbean 2.0.12 2022-07-13 00:05:31 -07:00
Justine Tunney e3cd476a9b Improve Lua and JSON serialization 2022-07-12 23:35:11 -07:00
Justine Tunney 3027d67037 Import more Musl math 2022-07-12 15:55:12 -07:00
Justine Tunney 6d52664aa7 Rename json test files to avoid build tuning 2022-07-12 13:24:25 -07:00
Justine Tunney 3f3e7e92d7 Audit every single JSON test 2022-07-12 12:40:38 -07:00
Justine Tunney 7965ed0232 Fix MODE=tiny build 2022-07-11 23:40:34 -07:00
Justine Tunney 6ee18986e4 Further improve JSON serialization 2022-07-11 23:06:49 -07:00
Justine Tunney 4814b6bdf8 Use ARM's faster math functions on non-tiny builds 2022-07-11 18:34:10 -07:00
Gautham 3c10fb5580 Fix quote escapes in JSON tests (#481) 2022-07-11 18:04:51 -07:00
William Chargin 0272f638a5 printimage: keep aspect ratio when scaling to fit (#479)
When `printimage` is invoked without `-w` or `-h`, we now preserve the
aspect ratio of each input image when scaling it to fit in the window.
A new flag `-i` ignores the aspect ratio, recovering the old behavior
when neither `-w` nor `-h` is passed. When `-i` is passed alongside
exactly one of `-w` or `-h`, the other dimension is just taken from the
window size, ignoring aspect ratio.

We also unconditionally print a newline between images to prevent them
from overlapping.

wchargin-branch: printimage-fit
wchargin-source: 4c2cfcffe9ce1a3b30c0ff051e3c6a2c166ae1c7
2022-07-11 15:58:21 -07:00
Gautham 9308463b0f Escape bad unicode in json tests (#480) 2022-07-11 15:16:44 -07:00
Gautham 594615a0ff Add more json tests for ljson (#478) 2022-07-11 11:54:24 -07:00
Gautham 11a1c62d11 Reduce JSON depth limit to 128 (#477)
- also remove tpyo from net.mk
- STACK_FRAME_UNLIMITED is not needed
- also let parser allow 0e1, 0e+1 as floats
2022-07-11 10:23:44 -07:00
Gautham aeee2f9106 Add depth limit for JSON (#476) 2022-07-11 08:17:58 -07:00
Justine Tunney 694a0da990 Make function call tracing lockless 2022-07-11 08:04:58 -07:00
Justine Tunney 9877c04fac Fix MODE=tiny build 2022-07-11 06:11:39 -07:00
Justine Tunney 3f015b1e51 Make some minor fixups to bug reporting, etc. 2022-07-11 05:58:24 -07:00
ProducerMatt 84764ce7b8 lre_test.lua: Benchmark() fails on some systems (#474) 2022-07-10 23:05:50 -07:00
Justine Tunney 68ca49bfdd Improve APE install scripts and add uninstaller
See #350 thanks @tkchia
2022-07-10 14:13:45 -07:00
Justine Tunney 331fdd1d29 Fix some broken builds and broken hearts 2022-07-10 09:08:38 -07:00
Gautham c0b325bafa Add json.org tests for DecodeJson (#473) 2022-07-10 08:35:51 -07:00
Justine Tunney 5fa77f1e8f Make _Thread_local more seamlessly working 2022-07-10 08:27:50 -07:00
Justine Tunney 5f4f6b0e69 Make _Thread_local work across platforms
We now rewrite the binary image at runtime on Windows and XNU to change
mov %fs:0,%reg instructions to use %gs instead. There's also simpler
threading API introduced by this change and it's called _spawn() and
_join(), which has replaced most clone() usage.
2022-07-10 04:01:17 -07:00
Justine Tunney e4d6e263d4 Rename ParseJson() to DecodeJson() for consistency 2022-07-09 17:34:41 -07:00
Justine Tunney 28f0104330 Bump redbean up to 2.0.10 2022-07-09 17:28:14 -07:00
William Chargin 4a1419fefa ttyraster: restore colors at start of each line (#471)
The patch in #469 was buggy for images where the start of a row matched
the end of the previous row. We don't re-issue the `setbgfg` ANSI codes
when we think that the color hasn't changed. But by sending an `\e[0m`
sequence at the end of the line without updating `bg` or `fg`, we
desynced `bg` and `fg` from the actual ANSI state. Now, we simply follow
that line-terminating `\e[0m` with another `setbgfg` call.

This bug was visible in images with a constant-color matte border:

![Screenshot of `printimage` output before and after this commit when
run on `lemurs.png` and `lemursborder.png`, where `lemursborder.png` has
a thick blue border around the outside of the image. Both versions look
fine for `lemurs.png`. For `lemursborder.png`, the "before" version has
a chess board pattern for the left border (except for the first row) and
the bottom border. The "after" version looks correct.][ss]

[ss]: https://user-images.githubusercontent.com/4317806/178120511-c1b89348-2376-4bf2-a2d3-8723d2663bd4.png

Fixes: 85aecbda67 ("ttyraster: reset ANSI attributes after each line (#469)")
wchargin-branch: ttyraster-restore-each-line
wchargin-source: 621a788cfa0a87ce360e142a6004e325cca70caa
2022-07-09 16:47:42 -07:00
Justine Tunney ee82cee432 Further improve json / lua serialization 2022-07-09 16:27:26 -07:00
Paul Kulchenko 9e86980191 Redbean fix listen leak (#470)
* Fix release of resources allocated to skipped sockets
* Update log messages for consistency
2022-07-09 12:16:44 -07:00
Justine Tunney 7756e27993 Fix GitHub build 2022-07-09 12:00:57 -07:00
Justine Tunney 2189877856 Implement new JSON parser for redbean 2022-07-09 11:44:19 -07:00
Justine Tunney d37536bd4b Fix oops w/ array serialization ordering 2022-07-09 10:29:26 -07:00
Justine Tunney 727d9cbf56 Bump redbean to 2.0.9 2022-07-09 05:50:05 -07:00
Justine Tunney 896db8843f Loosen restriction on valid hostnames
This changes fixes an issue with GitPod integration, where large numbers
in a hostname were incorrectly causing redbean to refuse a request.
2022-07-09 04:26:29 -07:00
Justine Tunney c9e68b0ebc Make redbean serialization deterministic 2022-07-09 04:09:51 -07:00
William Chargin 85aecbda67 ttyraster: reset ANSI attributes after each line (#469) 2022-07-09 01:38:35 -07:00
William Chargin a9145e7f1e printimage: keep aspect ratio with -w xor -h (#468) 2022-07-09 01:38:13 -07:00
Justine Tunney 182d7720cf Fix GitHub actions
Our build was flaking due to ETXTBSY errors running multiple redbean
instances in parallel. This is due to the StoreAsset() support which
seems to cause enough problems it's worth making a *breaking change*
turning it off by default for now. There's a new -* flag, to restore
redbean's old self-modifying behavior.
2022-07-09 01:18:55 -07:00
Justine Tunney 1c83670229 Write more redbean unit tests
- Fix DescribeSigset()
- Introduce new unix.rmrf() API
- Fix redbean sigaction() doc example code
- Fix unix.sigaction() w/ more than two args
- Improve redbean re module API (non-breaking)
- Enhance Lua with Python string multiplication
- Make third parameter of unix.socket() default to 0
2022-07-08 23:10:02 -07:00
Justine Tunney c5b9902ac9 Add ansi terminal audio prototype
It's never worked very well having nesemu1.com and printvideo.com
spawning an ffmpeg or sox subprocess and streaming audio samples via
pipes. Since these programs don't work very well for that purpose, and
if you're SSH'ing into the cloud, the speaker could be very far away.

This change is part of an experiment to instead patch desktop terminals
such as PuTTY, KiTTY, gnome-terminal, etc. to support receiving inband
audio samples as ANSI code, and then playing them on the speakers of the
local machine that's being used. This way we can use printf() as a cross
platform audio playback library.
2022-07-08 15:11:46 -07:00
Justine Tunney 50a6df89b8 Fix Lua type of zero
This change fixes a regression from 281a0f27 which resulted in the
integer literal `0` being interpreted as a floating point number. This
should also fix a reported issue with Fennel integration.
2022-07-08 10:12:19 -07:00
Justine Tunney fe5c475f83 Add Deflate() / Inflate() to redbean and fix bugs
The Compress() and Uncompress() APIs were a mistake. The functions
themselves work fine, but it's a design blemish and does superfluous
work. Since they were only introduced in the last few weeks, they're now
deprecated and references to them have been scrubbed from the website
and other documentation. Please use the new APIs since the old APIs will
be removed at some point in the future.

This change introduces automated Lua unit tests for the Redbean APIs.
There's a few functions that were broken which have now been fixed, e.g.
Underlong() and Decimate().
2022-07-08 09:47:04 -07:00
Paul Kulchenko a18044c504 Add OnServerListen hook to configure listen() (#459) 2022-07-08 07:17:25 -07:00
Paul Kulchenko 11ac8f11a9 Reduce redbean logging for EBADF (#461)
This may happen on explicitly closed client connections, so reduce the
logging level similar to other errors.
2022-07-08 07:14:50 -07:00
Paul Kulchenko 059fe22ea3 Improve redbean fetch() (#460)
* Updated Fetch to return `nil,error` on errors
* Fix localhost connect when only IP address is specified
2022-07-08 07:13:51 -07:00
Justine Tunney 853b6c3864 Improve system calls
- Wrap clock_getres()
- Wrap sched_setscheduler()
- Make sleep() api conformant
- Polyfill sleep() using select()
- Improve clock_gettime() polyfill
- Make nanosleep() POSIX conformant
- Slightly improve some DNS functions
- Further strengthen pledge() sandboxing
- Improve rounding of timeval / timespec
- Allow layering of pledge() calls on Linux
- Polyfill sched_yield() using select() on XNU
- Delete more system constants we probably don't need
2022-07-08 06:42:03 -07:00
Paul Kulchenko 5df3e4e7a8 Redbean zip and unix.poll doc update (#457) 2022-06-30 17:41:51 -07:00
Joseph Stewart 2eebb198a9 Add antirez picol (tiny Tcl clone) (#340) 2022-06-29 08:52:22 -07:00
Daniil Kulchenko 9f8e6c10dd Work around Rosetta clobbering startup registers on M1 Macs (issue #429) (#453)
Rosetta doesn't correctly respect the startup registers as defined in LC_UNIXTHREAD
which makes platform detection go awry. But at least Rosetta appears to consistently
set rbx to 0x00000000ffffffff and rdx to 0x0000000000000001 at startup for every
x64 executable I could get my hands on. So we use that to detect Rosetta's presence
and set up the correct registers for XNU.
2022-06-27 16:28:59 -07:00
Justine Tunney 5297897ba1 Add fixups for previous change 2022-06-27 15:00:51 -07:00
Justine Tunney 3c92adfd6e Strengthen the pledge() polyfill 2022-06-27 13:02:17 -07:00
Gautham a6f65eea7c Scan fixed-width integers in vcscanf (#424)
When a format string like %2x is provided, the width parameter was read
correctly as 2, but it was not used when decoding the number from the
input string (ie instead of reading 2 characters from the input string,
vcscanf read all the characters).

This change uses the value of width within the number decoding loop to
read fixed number of digits correctly. if width is zero (not provided),
the default of width is set as bits.

Fixes #423
2022-06-26 21:27:07 -07:00
Paul Kulchenko 32eec7df4c Update FATALF to exit without stack trace (#452) 2022-06-26 21:25:02 -07:00
Paul Kulchenko 283729dfe0 Redbean report write lock failure (#450)
This change updates redbean to report a non-writable path
when using StoreAsset, instead of crashing the worker.
2022-06-26 21:16:13 -07:00
Paul Kulchenko b4e9a77e74 Improve redbean ZeroBrane integration (#451) 2022-06-26 21:13:53 -07:00
Justine Tunney f6dab99f44 Release redbean 2.0.8 2022-06-26 17:25:54 -07:00
Gautham b535937fca Add sys.meta_path entry for APE zip store (#425) 2022-06-26 05:51:00 -07:00
Justine Tunney 893cc06fc2 Fix libunwind stubs 2022-06-26 05:29:25 -07:00
Justine Tunney 5cd0e876bd Scrub double-conversion and add quick wrapper 2022-06-26 05:03:25 -07:00
Gautham b1de862e16 Add double-conversion to third_party (#447) 2022-06-26 03:41:55 -07:00
Justine Tunney 2f515ac5e3 Fix code completion bug in Lua REPL 2022-06-26 03:11:56 -07:00
Justine Tunney 61257d48d4 Make some quick fixes and cleanup 2022-06-26 02:58:36 -07:00
Theta Nil fa5c2a24b5 Add tiny / opt to CI matrix (#448) 2022-06-25 22:08:41 -07:00
Justine Tunney fbc053e018 Make fixes and improvements
- Introduce __assert_disable global
- Improve strsignal() thread safety
- Make system call tracing thread safe
- Fix SO_RCVTIMEO / SO_SNDTIMEO on Windows
- Refactor DescribeFoo() functions into one place
- Fix fork() on Windows when TLS and MAP_STACK exist
- Round upwards in setsockopt(SO_RCVTIMEO) on Windows
- Disable futexes on OpenBSD which seem extremely broken
- Implement a better kludge for monotonic time on Windows
2022-06-25 21:09:09 -07:00
Gautham 5d837c4e7c Add complex math from musl (#422) 2022-06-23 18:56:10 -07:00
Paul Kulchenko 1bce1ca2e0 Polish redbean APIs and docs (#446)
* Remove undocumented ProgramSslCompression
* Rename GetLastModified to GetAssetLastModified
* Rename IsCompressed to IsAssetCompressed
* Reorganize currently deprecated functions
* Update sockopt documentation for clarity
2022-06-23 18:54:35 -07:00
Justine Tunney a9d77362f9 Release redbean 2.0.7 2022-06-23 18:14:17 -07:00
Justine Tunney 4b9ee980a1 Release redbean 2.0.6 2022-06-23 17:37:55 -07:00
Justine Tunney e55ef556aa Double default build process quota
See #430
Fixes #438
2022-06-23 16:46:12 -07:00
Tom Gillespie 7f26779d6f Make build process quota overridable (#430)
P had the same assignment as F. This is now fixed so that it is
possible to increase RLIMIT_NPROC (e.g. when trying to build
on system accounts with more than 1024 processes running).
2022-06-23 16:40:47 -07:00
Justine Tunney 3d07f362d7 Release redbean 2.0.5 2022-06-23 16:09:20 -07:00
Justine Tunney f79b2fe23a Change assumptions about PROT_EXEC implying PROT_READ
On all operating systems tested so far, PROT_EXEC without PROT_READ
always makes memory readable. This turned out to not be the case on
Chromebooks, which likely means they have the capability of running
programs which aren't able to read their own code.
2022-06-23 14:13:19 -07:00
Justine Tunney e4258015b7 Improve detection of getrandom / getentropy
It's been reported that the 500th system call getentropy() isn't present
on Darwin Kernel Version 15.6.0 virtual machines. We work around this by
ignoring SIGSYS temporarily.
2022-06-23 14:09:32 -07:00
Theta Nil 49905fed0b Fix CI badge in README (#443) 2022-06-23 13:22:28 -07:00
Justine Tunney 44da16255a Show crash reports on SIGSYS
This will help make it easier to troubleshoot ABI breakages with on
operating systems that, unlike Linux don't have ironclad guarantees
to not break userspace.
2022-06-23 13:01:01 -07:00
Justine Tunney a4601a24d3 Perform some code cleanup 2022-06-23 10:21:07 -07:00
Theta Nil 0dd9629562 Fix GitHub Actions build (#442) 2022-06-23 07:04:59 -07:00
Justine Tunney 3fc23c6cbc Add getcontext() and setcontext()
Fixes #413
2022-06-23 06:01:44 -07:00
Theta Nil 2b54f1bcf6 Migrate from Travis to GitHub Actions (#441) 2022-06-23 04:39:00 -07:00
Justine Tunney 206f073181 Add stat mode macros to redbean unix api 2022-06-23 04:05:51 -07:00
Justine Tunney 17cbe73411 Add finger demo to redbean and fix regression
This change fixes a regression in unix.connect() caused by the recent
addition of UNIX domain sockets. The BSD finger command has been added
to third_party for fun and profit. A new demo has been added to redbean
showing how a protocol as simple as finger can be implemented.
2022-06-23 03:42:05 -07:00
Justine Tunney 2415afab0e Don't use vfork() on XNU (#426) 2022-06-23 01:18:47 -07:00
Justine Tunney 1235ad6006 Add temporary stubs for libunwind 2022-06-22 07:11:08 -07:00
Justine Tunney 9545062175 Add pthread tls api 2022-06-22 07:08:17 -07:00
Justine Tunney 0e773e23ad Add special thanks section to readme 2022-06-22 04:35:43 -07:00
Justine Tunney fc097ac275 Add unix domain socket support to redbean 2022-06-22 03:04:25 -07:00
Jules Bertholet 4b23985b7f Reformat libc README.md (#437) 2022-06-21 07:50:42 -07:00
Justine Tunney ff28e38ef1 Close more superfluous file descriptors
GitHub Actions leaks file descriptors with numbers greater than a
hundred. But our test runner only checked for FDs up to 64.
2022-06-21 07:42:07 -07:00
Justine Tunney b8e9f71d33 Add futexes for OpenBSD 2022-06-21 07:17:05 -07:00
Justine Tunney 1216b6e7c7 Add xadd to chibicc 2022-06-21 04:55:27 -07:00
Justine Tunney 56eff8544f Fix V=0 make mode
See #431
2022-06-21 01:32:31 -07:00
Justine Tunney f317a47cd8 Fixup and polish chibicc some more
Fixes #433
2022-06-20 04:32:25 -07:00
Justine Tunney a988896048 Add atomics to chibicc
This change also fixes #434 and makes the chibicc assembler better.
2022-06-20 03:08:00 -07:00
Justine Tunney 5ddf43332e Release redbean 2.0.4 2022-06-19 20:17:58 -07:00
Justine Tunney d5312b60f7 Make improvements to locking
This change makes pthread_mutex_lock() as fast as _spinlock() by
default. Thread instability issues on NetBSD have been resolved.
Improvements made to gdtoa thread code. Crash reporting will now
synchronize between threads in a slightly better way.
2022-06-19 01:30:12 -07:00
Justine Tunney 25041b8026 Remove unsafe emacs config from tidy 2022-06-19 00:59:58 -07:00
Justine Tunney 2c7fe0f361 Fix redbean regression from previous commit 2022-06-19 00:59:07 -07:00
Justine Tunney 8c895453e2 Fix build break due to deleted file 2022-06-18 01:55:32 -07:00
Justine Tunney c1cfca8ae1 Make fixes and improvements
- Polyfill UTIME_OMIT on XNU
- Refactor Lua build code so it's better
- Add unix module to lua.com (Discord request)
- Add unix.utimensat() and unix.futimens() to redbean
- Avoid creating double slash path in linenoise (#428)
- Remove double slashes in NT paths automatically (#428)
- Make strerror() smarter about showing NT errors (#428)

Fixes #428
2022-06-18 01:46:12 -07:00
Justine Tunney 67b28b9af1 Update redbean documentation 2022-06-17 08:05:47 -07:00
Justine Tunney 52ed099fe6 Favor $HOME directory over /tmp
This should strike a better compromise that keeps people happy about the
security of APE when it extracts the loader. This way systems with users
who aren't trusted (e.g. CPanel) won't be at any risk of compromise when
there isn't an `ape` loader on the system `$PATH`.

This change also bumps redbean up to 2.0.3
2022-06-17 07:56:54 -07:00
Justine Tunney 5e04827949 Add fileio and sqlar SQLite extensions to redbean 2022-06-17 06:04:09 -07:00
Justine Tunney bc90db0d4b redbean 2.0.2 2022-06-17 04:18:20 -07:00
Justine Tunney 7b34f32fc4 Use exponential backoff in mutexes 2022-06-17 04:17:21 -07:00
Justine Tunney a0ddb889da Polyfill sched_yield() on XNU
We were using the Mach system call swtch() earlier. It's possible Apple
removed this system call in their recent 12.4 upgrade. We're better off
using x86 PAUSE here, since Mach is less public than the UNIX syscalls.

See #426
2022-06-17 03:48:15 -07:00
Justine Tunney 34e39ad027 Enable sqlite zipfile module in redbean
This change also breaks out a bunch of extension files that the SQLite
authors inlined into a shell.c amalgamation.
2022-06-17 02:49:04 -07:00
Justine Tunney 2c7f865b12 Improve http caching in redbean 2022-06-17 02:44:15 -07:00
Justine Tunney d0d9cd38c5 Write tests and fixes for utimensat() 2022-06-17 02:43:00 -07:00
Justine Tunney c06ffd458c Write some lock contention tests 2022-06-16 09:06:09 -07:00
Justine Tunney 42c38bc3e3 Delete some files we don't need 2022-06-16 09:04:37 -07:00
Justine Tunney fae2a17d2c Fix build breakage in MODE=dbg 2022-06-15 19:37:39 -07:00
Justine Tunney e466dd0553 Add torture test for zipos file descriptors
This change hardens the code for opening /zip/ files using the system
call interface. Thread safety and signal safety has been improved for
file descriptors in general. We now document fixed addresses that are
needed for low level allocations.
2022-06-15 16:29:49 -07:00
Justine Tunney 579080cd4c Improve generation of redbean static headers
This change fixes an issue where Cache-Control could be outputted twice
when Lua code calls SetHeader() from OnHttpRequest().
2022-06-14 22:01:13 -07:00
Justine Tunney 42b34c26f8 Release redbean 2.0.1 2022-06-14 20:21:34 -07:00
Paul Kulchenko 8b13e1acc8 Fixed redbean compression functions (#421) 2022-06-14 17:50:25 -07:00
Paul Kulchenko 58c77a9b1d Fix recent regression in redbean helpers (#419) 2022-06-14 17:22:51 -07:00
ProducerMatt bba7f5e296 Extend quotas for various slow tests (#420) 2022-06-14 17:21:57 -07:00
Justine Tunney 1c387727fe Add long double printf formatting support
You can now easily print numbers that have 80 bits of precision. This
change also imports many of the test vectors from the gdtoa codebase.
2022-06-14 02:18:03 -07:00
Gautham dae0583ee5 Add pragma to ignore -Wundef (#418)
also ignore Wc++-compat when not in use
2022-06-13 23:13:51 -07:00
Justine Tunney a3865ecc3c Make more fixes and improvements
- Fix Makefile flaking due to ZIPOBJ_FLAGS generation
- Make printf() floating point and gdtoa thread safe
- Polish up the runit / runitd programs some more
- Prune some more makefile dependencies
2022-06-13 11:02:13 -07:00
Justine Tunney 3c285337a2 Fix %c with nul character
Fixes #417
2022-06-12 22:28:19 -07:00
Justine Tunney 91953dd308 Add some more necessary locks 2022-06-12 22:20:59 -07:00
Justine Tunney 8b72490431 Make mutex calling code 10x tinier
Calls to lock/unlock functions are now NOPs by default. The first time
clone() is called, they get turned into CALL instructions. Doing this
caused funcctions like fputc() to shrink from 85 bytes to 45+4 bytes.
Since the ANSI solution of `(__threaded && lock())` inlines os much
superfluous binary content into functions all over the place.
2022-06-12 20:17:12 -07:00
Justine Tunney 8cdec62f5b Apply even more fixups
- Finish cleaning up the stdio unlocked APIs
- Make __cxa_finalize() properly thread safe
- Don't log locks if threads aren't being used
- Add some more mutex guards to places using _mmi
- Specific lock names now appear in the --ftrace logs
- Fix mkdeps.com generating invalid Makefiles sometimes
- Simplify and fix bugs in the test runner infrastructure
- Fix issue where sometimes some functions wouldn't be logged
2022-06-12 11:57:00 -07:00
Justine Tunney 4ddfc47d6e Make some more fixups 2022-06-12 09:37:17 -07:00
Justine Tunney 6070a53e89 Make a few fixups 2022-06-12 07:32:14 -07:00
Justine Tunney a5849f8549 Improve redbean wildcard certificate support 2022-06-11 19:25:03 -07:00
Justine Tunney 29af890efa Disable malloc mutex when not using threads 2022-06-11 19:18:09 -07:00
Justine Tunney 517267a577 Reduce MKDEPS.COM latency from 60ms to 18ms 2022-06-11 12:23:47 -07:00
Justine Tunney e96aceae41 Bump support up to FreeBSD 13 and NetBSD 9.2
These releases are really exciting since they contained the patches we
worked to get upstreamed. It means that their /bin/sh interpreters all
work fine with Actually Portable Executable now.
2022-06-11 10:36:17 -07:00
Justine Tunney 1ae6cfc198 Update build binaries for WSL
If you use WSL then you may want to delete /tmp/ape, o/tmp/ape, and
/usr/bin/ape before running make again. That should resolve issues.
2022-06-11 05:25:03 -07:00
Justine Tunney e32288d4ea Fix WSL regression with APE loader
Microsoft's version of Linux requires that the ELF program headers say
that the executable has 4096 byte alignment. However, it doesn't force
us to pad the binary with NOPs to a page-aligned size.
2022-06-11 05:17:26 -07:00
Justine Tunney c260345e06 Make locks more reliable
This change switches most of the core locks to be re-entrant, in order
to reduce the chance of deadlocking code that does, clever things with
asynchronous signal handlers. This change implements it it in pthreads
so we're one step closer to having a standardized threading primitives
2022-06-11 02:07:20 -07:00
Justine Tunney 5ea618f0af Make clock_gettime() faster on FreeBSD 2022-06-11 00:05:06 -07:00
Justine Tunney 5deda43766 Handle wildcard certificates in redbean 2022-06-10 21:59:14 -07:00
Justine Tunney c6d8e516b2 Print warning when microbenchmarking w/ powersave
RDTSC on Linux has so much jitter when the CPU is in powersave mode
causing things like microbenchmarks to have a 1000% margin of error
2022-06-10 21:07:03 -07:00
Justine Tunney 41c86fe86b Fix build error in MODE=dbg 2022-06-10 20:25:29 -07:00
Justine Tunney 8a967ae6fe Fix header dependencies in third_party/make 2022-06-10 13:07:11 -07:00
Justine Tunney af4687cc3f Add Info-ZIP unzip.com 2022-06-10 05:09:11 -07:00
Justine Tunney 1531f1a4a7 Add CTRL-U shortcut to QuickJS REPL 2022-06-10 03:39:05 -07:00
Justine Tunney fde939291d Fix an UBSAN warning 2022-06-10 03:33:05 -07:00
Justine Tunney f8520e10b2 Add ProgramMaxPayloadSize() API to Redbean
This way we don't have to use the flag and can configure it using
.init.lua instead.
2022-06-10 03:28:46 -07:00
Justine Tunney 2528536cff Fix some plinko documentation 2022-06-10 03:28:24 -07:00
Justine Tunney a41669dec6 Make dlmalloc a little faster
This change also documents the libc arena allocator.
2022-06-09 21:25:21 -07:00
Justine Tunney fa1e8a3e65 Make emacs not croak when editing dlmalloc 2022-06-09 17:45:30 -07:00
Justine Tunney 3c7ae0fc72 Add w3c html tidy 2022-06-09 06:51:08 -07:00
Justine Tunney ecc8962555 Improve performance of runit.com
We now have O(1) deflates rather than O(n).
2022-06-08 21:52:47 -07:00
Justine Tunney e1a40783da Fix fadvise() on Windows 2022-06-08 20:33:13 -07:00
Justine Tunney adac64a52b Reduce makefile dependencies by 10%
The includes in libc/calls/calls.h have now been refactored so that
functions with struct parameters are declared in libc/calls/struct/
2022-06-08 20:01:28 -07:00
Justine Tunney 4e7ce1538c Fix linenoise flickering on CTRL+L 2022-06-08 18:40:51 -07:00
Justine Tunney 701564de19 Perform better fixups of NT paths in execve()
This change ensures we do a better job translating /c/foo.bar paths into
c:/foo.bar paths on Windows when generating the CreateProcess() cmd line
thus fixing a regression that happened in the last two months when using
the help() feature of Actually Portable Python in the CMD.EXE shell.
2022-06-08 18:27:19 -07:00
Justine Tunney cfb5d3e406 Fix issue with --strace log 2022-06-08 17:20:19 -07:00
Justine Tunney e011973593 Remove -no-canonical-prefixes flag 2022-05-29 19:14:56 -07:00
Justine Tunney da6d610056 Use Lua auto buffers when possible 2022-05-29 19:04:30 -07:00
Justine Tunney 13ee75150c Improve redbean plus code size optimizations
This change turns symbol table compression back on using Puff, which
noticeably reduces the size of programs like redbean and Python. The
redbean web server receives some minor API additions for controlling
things like SSL in addition to filling gaps in the documentation.
2022-05-29 08:21:19 -07:00
Justine Tunney 425ff5dff0 Refactor some code
- Write tests for cthreads
- Fix bugs in pe2.com tool
- Fix ASAN issue with GetDosEnviron()
- Consolidate the cthread header files
- Some code size optimizations for MODE=
- Attempted to squash a tls linker warning
- Attempted to get futexes working on FreeBSD
2022-05-28 14:09:21 -07:00
Gautham 909e54510d Fix Python test_repl in MODE=dbg (#412) 2022-05-28 05:50:42 -07:00
Justine Tunney de5de19004 Make improvements
- Document redbean's argon2 module
- Fix regressions in cthreads library
- Make testlib work better with threads
- Give the cthreads library lots of love
- Remove some of the stdio assembly code
- Implement getloadavg() across platforms
- Code size optimizations for errnos, etc.
- Only check for signals in main thread on Windows
- Make errnos for dup2 / dup3 consistent with posix

This change also fixes a bug in the argon2 module, where the NUL
terminator was being included in the hash encoded ascii string. This
shouldn't require any database migrations to folks who found this module
and productionized it, since the argon2 library treats it as a c string.
2022-05-28 00:28:09 -07:00
Justine Tunney cb67223051 Add malloc logging tool
STATIC_YOINK("enable_memory_log");
2022-05-26 23:19:22 -07:00
Gautham 7e9fb0a9f1 Move importlib functions to within C (#408)
This offers a 10% speedup in Python startup time. It also
makes debugging using cosmopolitan tooling easier.
2022-05-26 23:20:59 -07:00
Justine Tunney 10b97ca630 Add test for sendfile() and reduce branches 2022-05-25 22:29:10 -07:00
Justine Tunney 8f12cd980d Fix execution bug w/o binfmt_misc
The longjmp relocation in the master boot record coincidentally had a
quote character in it, which caused the master boot record to be used
when interpreter by the shell. The solution, is to move the grub stub
below the long mode loader so the real mode loader relocation is near
the master boot record. This change includes a regression test.
2022-05-25 17:24:02 -07:00
Justine Tunney 1a29424c52 Fix ARG_MAX with chibicc documentation generator 2022-05-25 14:06:11 -07:00
Justine Tunney d230a01222 Make build hermetic without shell scripts
- Fix some minor issues in ar.com
- Have execve() look for `ape` command
- Rewrite NT paths using /c/ rather /??/c:/
- Replace broken GCC symlinks with .sym files
- Rewrite $PATH environment variables on startup
- Make $(APE_NO_MODIFY_SELF) the default bootloader
- Add all build command dependencies to build/bootstrap
- Get the repository mostly building from source on non-Linux
2022-05-25 13:55:57 -07:00
Justine Tunney d44ff6ce1f Make improvements
- Implement openpty()
- Add `--assimilate` flag to APE bootloader
- Restore Linux vDSO clock_gettime() support
- Use `$(APE_NO_MODIFY_SELF)` on more programs
2022-05-24 10:58:48 -07:00
Justine Tunney cef50f2a6b Write test for stack overflow detection 2022-05-23 16:32:16 -07:00
Justine Tunney 99e67c348b Reduce Makefile dependencies by 4% 2022-05-23 15:07:01 -07:00
Justine Tunney e8632cfc30 Revert recent change to ASAN module 2022-05-23 13:30:16 -07:00
Justine Tunney 312ed5c67c Fix some issues and do some code cleanup 2022-05-23 10:15:53 -07:00
Justine Tunney 1f229e4efc Use re-entrant locks on stdio 2022-05-22 08:28:33 -07:00
Justine Tunney 4e9662cbc7 Write tests for new APE loader and fix bugs
- Add FreeBSD-specific mmap() flags
- Reduce size of the APE loader from 8kb to 4kb
- Work towards fixing the Makefile build on WSL
- Automate testing of APE no-modify-self behaviors
- Make the ape.S shell script code cleaner and tinier
- Improve the APE sanity check to test behavior better
- Fixed issue with ShowCrashReports() sigaltstack() on BSDs
- Delete symbols for S_MODE magnums which wasted compile time

If you checked out yesterday's APE commit, please run:

    rm -f /usr/bin/ape o/tmp/ape /tmp/ape "${TMPDIR:-/tmp}/ape"

Because this change fixes certain aspects of the new ABI. We don't have
automated migrations for APE loader versions yet. Thanks! You can also
download prebuilt binaries here:

- https://justine.lol/ape.elf    (Linux/FreeBSD/NetBSD/OpenBSD)
- https://justine.lol/ape.macho  (Apple)

Install the appropriate one as `/usr/bin/ape`.
2022-05-22 05:45:38 -07:00
ProducerMatt 056dc5f554 Extend quotas for various slow tests (#411) 2022-05-21 20:13:01 -07:00
Justine Tunney db0d8dd806 Support Linux binfmt_misc and APE loading on Apple
The "no modify self" variant of Actually Portable Executable is now
supported on all platforms. If you use `$(APE_NO_MODIFY_SELF)` then
ld.bfd will embed a 4096 byte ELF binary and a 4096 byte Macho file
which are installed on the fly to ${TMPDIR:-/tmp}, which enables us
launch the executable, without needing to copy the whole executable

To prevent it from copying a tiny executable to your temp directory
you need to install the `ape` command (renamed from ape-loader), to
a system path. For example:

    # FreeBSD / NetBSD / OpenBSD
    make -j8 o//ape/ape
    cp o//ape/ape /usr/bin/ape

    # Mac OS
    # make -j8 o//ape/ape.macho
    curl https://justine.lol/ape.macho >/usr/bin/ape
    chmod +x /usr/bin/ape

On Linux you can get even more performance with the new binfmt_misc
support which makes launching non-modifying APE binaries as fast as
launching ELF executables. Running the following command:

    # Linux
    ape/apeinstall.sh

Will copy APE loader to /usr/bin/ape and register with binfmt_misc
Lastly, this change also fixes a really interesting race condition
with OpenBSD thread joining.
2022-05-21 09:28:25 -07:00
Justine Tunney 7838edae88 Fix many thread and file descriptor issues on Windows
The greenbean web server now works nearly perfectly on Windows with over
1000 threads. But some synchronization issues still remain which prevent
us from going over nine thousand.
2022-05-20 20:57:19 -07:00
Justine Tunney 96781d0679 Fix mistake with previous commit 2022-05-20 15:08:02 -07:00
Justine Tunney f4fc16754a Fix build invalidation issue with GNU Make
The `HDRS` and/or `INCS` variables need to be defined in order for
`mkdeps.com` to compute build dependencies for `o//depend`. We're using
INCS here since makeint.h is the evil kind of header that isn't actually
defining a library interface but rather is a hodgepodge of common code.
2022-05-20 14:20:54 -07:00
Justine Tunney 8141988e92 Shave 4kb off default mode binaries 2022-05-20 08:18:20 -07:00
Gautham fb4382e9a4 os.realname variable for correct USER_SITE value (#410)
In site.py, Python uses os.name to decide where the USER_SITE (ie the
folder containing the user's locally installed packages) is located.
With cosmo we have set os.name as "posix" always, so we use a new
os.realname to decide the USER_SITE location.
2022-05-20 08:17:10 -07:00
Justine Tunney c8a2f04058 Reduce ftrace overhead to 280ns 2022-05-20 04:46:42 -07:00
Justine Tunney 4245da19e2 Remove some old dead code from ftrace 2022-05-20 04:01:50 -07:00
Justine Tunney b4ae385a6a Fix amalgamation warnings 2022-05-19 19:08:35 -07:00
Justine Tunney c21412bc2c Remove carriage return from function traces 2022-05-19 17:46:14 -07:00
Justine Tunney ec2cb88058 Make fixes and improvements
- Document more compiler flags
- Expose new __print_maps() api
- Better overflow checking in mmap()
- Improve the shell example somewhat
- Fix minor runtime bugs regarding stacks
- Make kill() on fork()+execve()'d children work
- Support CLONE_CHILD_CLEARTID for proper joining
- Fix recent possible deadlock regression with --ftrace
2022-05-19 16:57:49 -07:00
Justine Tunney 6e52cba37a Fix stdio regression
This change fixes a nasty regression caused by
80b211e314 which deadlocked.

This change also causes MbedTLS to prefer the ChaCha ciphersuite on
older CPUs that don't have AES hardware instructions.
2022-05-19 00:51:15 -07:00
Justine Tunney 9208c83f7a Make some systemic improvements
- add vdso dump utility
- tests now log stack usage
- rename g_ftrace to __ftrace
- make internal spinlocks go faster
- add conformant c11 atomics library
- function tracing now logs stack usage
- make function call tracing thread safe
- add -X unsecure (no ssl) mode to redbean
- munmap() has more consistent behavior now
- pacify fsync() calls on python unit tests
- make --strace flag work better in redbean
- start minimizing and documenting compiler flags
2022-05-18 16:52:36 -07:00
Justine Tunney c6bbca55e9 Disable Linux vDSO support for now
It's been encountering some strange errors that are difficult to
reproduce.
2022-05-17 12:51:25 -07:00
Justine Tunney b2c6ec6eab Make linenoise properly recover from backgrounding 2022-05-17 12:10:22 -07:00
Justine Tunney 2ad1c9078d Improve dependency generation for libcxx headers
This change also updates redbean to have more informative error messages
based on the ones that are actually showing up in production.
2022-05-17 11:38:35 -07:00
Justine Tunney 3d99ebb68c Add raw parameter to redbean lua compress 2022-05-17 10:41:23 -07:00
Justine Tunney f37524ef4f Add ANSI mode guard to header 2022-05-17 07:43:10 -07:00
Justine Tunney 2743f3d012 Clean up the clone() code a bit 2022-05-17 07:40:00 -07:00
Justine Tunney ce71677156 Improve threading support further 2022-05-17 04:14:28 -07:00
Justine Tunney 8bfb70ca3f Add Compress() and Uncompress() to redbean 2022-05-16 16:49:20 -07:00
Justine Tunney 59b6ae1cbd Add NET_HTTP to the amalgamation
This also fixes some POSIX header warnings.
2022-05-16 14:47:49 -07:00
Justine Tunney 55de4ca6b5 Support thread local storage 2022-05-16 13:20:08 -07:00
Justine Tunney 91ee2b19d4 Polish greenbean example a bit more
Windows support for this example is still a work in progress. It's
encountering some unusual crashes. Thank you Chris Wellons for the cool
synchronization code too!
2022-05-15 09:19:49 -07:00
Paul Kulchenko e5e141d9b5 Update redbean unix module documentation for consistency (#409) 2022-05-14 23:25:54 -07:00
Justine Tunney c446af799e Compress network file transfers 2022-05-14 23:17:22 -07:00
Justine Tunney 6bf6f9e376 Add compress and decompress examples 2022-05-14 21:13:45 -07:00
Justine Tunney 54e6f564c1 Let ctrl-c interrupt lua server pages in repl mode 2022-05-14 11:47:16 -07:00
Justine Tunney 178a6da4b9 Add high performance threaded webserver example 2022-05-14 07:44:02 -07:00
Justine Tunney 80b211e314 Add raw memory visualization tool to redbean
This change introduces a `-W /dev/pts/1` flag to redbean. What it does
is use the mincore() system call to create a dual-screen terminal
display that lets you troubleshoot the virtual address space. This is
useful since page faults are an important thing to consider when using a
forking web server. Now we have a colorful visualization of which pages
are going to fault and which ones are resident in memory.

The memory monitor, if enabled, spawns as a thread that just outputs
ANSI codes to the second terminal in a loop. In order to make this
happen using the new clone() polyfill, stdio is now thread safe.

This change also introduces some new demo pages to redbean. It also
polishes the demos we already have, to look a bit nicer and more
presentable for the upcoming release, with better explanations too.
2022-05-14 04:33:58 -07:00
Justine Tunney 578cb21591 Add syscalls to Blinkenlights and fix bugs 2022-05-13 13:31:21 -07:00
Gautham f6df29cc3d Add Python JSON tests (#407) 2022-05-13 13:02:15 -07:00
Justine Tunney d25a67f4eb Fix cross-device link in Python tests 2022-05-13 06:50:55 -07:00
Justine Tunney 77b70ba193 Add fixes to previous commit 2022-05-13 05:05:27 -07:00
Gautham 83b743cf96 Python 3.7 METH_FASTCALL backport (#406) 2022-05-13 05:05:12 -07:00
Justine Tunney fec396037a Get threads working on all platforms
We now have a high-quality clone() implementation for creating
lightweight threads on Linux/Windows/FreeBSD/NetBSD/OpenBSD.
2022-05-13 03:59:09 -07:00
Justine Tunney 4e62cefa6e Add zlib gzip functions
These will now be included in the cosmopolitan.a releases. It took a bit
of time because, these functions depend on heavyweight parts of the libc
that wouldn't be appropriate for the core zlib library to depend upon.

Fixes #345
2022-05-12 12:17:40 -07:00
Justine Tunney 4499f98e76 Add /.args feature to Redbean/Lua/SQLite/Python/QuickJS
You now have some ability to truly make an executable yours, by adding a
`.args` file to the root of the zip structure. If this is specified,
then you'll be overriding the default CLI args.

This will be a great feature for folks who want to distribute their own
apps, using the interpreter executable, but have the executable appears
to be just your app rather than being the interpreter.
2022-05-12 11:04:47 -07:00
Justine Tunney 0f6251f4d2 Make redbean unix.open default to O_RDONLY 2022-05-12 09:43:01 -07:00
Justine Tunney ec6e0fa5f4 Wrap sched_getaffinity
Fixes #385
2022-05-12 07:22:57 -07:00
Justine Tunney 335d158ab6 Decrease strace verbosity 2022-05-12 06:58:51 -07:00
Justine Tunney 2ea1dc405c Revert "Backport METH_FASTCALL from Python 3.7 (#328)"
This reverts commit cf73bbd678.
2022-05-12 06:49:54 -07:00
Justine Tunney e7611a8476 Make improvements
- Get threads working on NetBSD
- Get threads working on OpenBSD
- Fix Emacs config for Emacs v28
- Improve --strace logging of sigset_t
- Improve --strace logging of struct stat
- Improve memory safety of DescribeThing functions
- Refactor auto stack allocation into LIBC_RUNTIME
- Introduce shell.com example which works on Windows
- Refactor __strace_thing into DescribeThing functions
- Document the CHECK macros and improve them in NDEBUG mode
- Rewrite MAP_STACK so it uses FreeBSD behavior across platforms
- Deprecate and discourage the use of MAP_GROWSDOWN (it's weird)
2022-05-12 06:45:36 -07:00
Justine Tunney dd9ab01d25 Revert "Use 64-bit years"
This reverts commit cfc3a953ae.
2022-05-12 06:45:36 -07:00
Gautham cf73bbd678 Backport METH_FASTCALL from Python 3.7 (#328) 2022-05-12 02:27:16 -07:00
Paul Kulchenko 70c97f598b Support redbean Fetch() headers (#405) 2022-05-11 22:06:42 -07:00
Justine Tunney cfc3a953ae Use 64-bit years
This change makes strftime() go faster and makes it possible to format
timestamps through the big bang to most of the stelliferous era. India
has also been added as a timezone to most binaries. Since we were able
to change the struct tm abi, this makes cosmopolitan libc superior, to
just about everything else, when it comes to standing the test of time
2022-05-11 17:58:56 -07:00
Justine Tunney 2aebda7718 Fix some bugs 2022-05-11 02:50:30 -07:00
Gautham 363d2ec436 Load Python C extensions of external packages correctly (#391) 2022-05-08 21:49:50 -07:00
Paul Kulchenko b30f5c0c4f Fix pledge to process lists with zero index syscalls (#402) 2022-05-06 20:55:28 -07:00
Justine Tunney 15c59e716f Fix build break in MODE=asan 2022-05-05 11:12:48 -07:00
Justine Tunney c9f966d73c Fix build suboptimality
Non-default build modes were having artifacts invalidated needlessly.
2022-05-04 22:51:51 -07:00
Justine Tunney 93906f8f28 Comment out psrldq_fuzz test
There's a mysterious error on Intel's low-power CPUs. It appears to be
triggered by the C code. Here's the vectors in CP-437:

    n=1
    u"x2ΩΦ▄█b∙¥¼U╞╣“┘$"
    u"2ΩΦ▄█b∙¥¼U╞╣“┘$ "
    u"2ΩΦ▄█b¥¼¼U╞╣“┘$ "
2022-05-04 21:43:38 -07:00
Justine Tunney 42071ffe6f Revert previous commit 2022-05-04 20:22:56 -07:00
Justine Tunney 854c90f547 Fix build flake on low-power processors 2022-05-04 19:53:28 -07:00
Justine Tunney a85406da4c Fix some example code 2022-04-29 08:19:09 -07:00
Justine Tunney e4b559c76a Simplify the redbean JSON encoder 2022-04-29 06:35:27 -07:00
Justine Tunney 2d1731b995 Polish redbean serialization 2022-04-29 06:10:10 -07:00
Justine Tunney 7aafa64ab3 Make improvements
- Bump redbean up to 2.0
- Trim down the MODE=tiny build a bit
- Add Indian Standard Time to zoneinfo
2022-04-29 00:42:47 -07:00
Justine Tunney c9a981fdbe Fix some more reported issues 2022-04-28 20:36:33 -07:00
Justine Tunney 80c4533303 Unbreak MODE=tiny build
This change also fixes a minor issue with pledge() polyfill introduced
the last change setting errno appropriately.

Fixes #396
2022-04-28 10:43:05 -07:00
Justine Tunney 47b3274665 Make improvements
- Add rusage to redbean Lua API
- Add more redbean documentation
- Add pledge() to redbean Lua API
- Polyfill OpenBSD pledge() for Linux
- Increase PATH_MAX limit to 1024 characters
- Untrack sibling processes after fork() on Windows
2022-04-28 09:57:07 -07:00
Paul Kulchenko 9a6bd304a5 Fix reporting of Lua stack items from various hook calls (#395)
Some hooks can be called after OnHttpRequest, which may leave an
anchored item on stack, so this have to be taken into account to
avoid spurious reports.
2022-04-27 21:57:52 -07:00
Justine Tunney e3a7ab1804 Fix some bugs
- addr2line backtrace should continue on eintr
- lua crashes if we try to iterate a non-table
2022-04-27 21:26:46 -07:00
Steve Phillips 92ebef16ee Add complex square root (#394) 2022-04-27 21:19:48 -07:00
Justine Tunney cc0d1ec076 Fix some bugs
- addr2line backtrace should continue on eintr
- lua crashes if we try to iterate a non-table
2022-04-27 20:18:34 -07:00
Justine Tunney 6a145a9262 Make improvements
- Add hierarchical auto-completion to redbean's repl
- Fetch latest localtime() and strftime() from Eggert
- Shave a few milliseconds off redbean start latency
- Fix redbean repl with multi-line statements
- Make the Lua unix module code more elegant
- Harden Lua data structure serialization
2022-04-27 05:39:39 -07:00
Justine Tunney d57b81aac7 Make improvements
- Add GetCpuCount() API to redbean
- Add unix.gmtime() API to redbean
- Add unix.readlink() API to redbean
- Add unix.localtime() API to redbean
- Perfect the new redbean UNIX module APIs
- Integrate with Linux clock_gettime() vDSO
- Run Lua garbage collector when malloc() fails
- Fix another regression quirk with linenoise repl
- Fix GetProgramExecutableName() for systemwide installs
- Fix a build flake with test/libc/mem/test.mk SRCS list
2022-04-26 16:46:15 -07:00
Paul Kulchenko 860ea18a87 Fix errno object handling in redbean (#392) 2022-04-26 05:20:53 -07:00
Paul Kulchenko e6fab847a0 Update redbean to stringify error objects (#393) 2022-04-25 21:14:36 -07:00
Justine Tunney 72e9be5c20 Update redbean lua example code 2022-04-25 09:31:28 -07:00
Justine Tunney 451e3f73d9 Improve redbean
- Improve serialization
- Add Benchmark() API to redbean
- Refactor UNIX API to be assert() friendly
- Make the redbean Lua REPL print data structures
- Fix recent regressions in linenoise reverse search
- Add -i flag so redbean can be a language interpreter
2022-04-25 08:30:14 -07:00
Justine Tunney 2046c0d2ae Make improvements
- Expand redbean UNIX module
- Expand redbean documentation
- Ensure Lua copyright is embedded in binary
- Increase the PATH_MAX limit especially on NT
- Use column major sorting for linenoise completions
- Fix some suboptimalities in redbean's new UNIX API
- Figured out right flags for Multics newline in raw mode
2022-04-24 10:06:05 -07:00
Justine Tunney cf3174dc74 Put redbean lua init demo code 2022-04-22 19:14:49 -07:00
Justine Tunney 2f56ebfe78 Do code cleanup use duff device linenoise i/o 2022-04-22 18:56:52 -07:00
Justine Tunney 6ff46ca373 Make it possible to compile redbean with chibicc
This cuts build latency down from 5 seconds to 500 milliseconds.
2022-04-22 15:25:04 -07:00
Justine Tunney 552525cbdd Fix a few reported issues 2022-04-21 22:07:21 -07:00
Justine Tunney a259e43d84 Add execve() test 2022-04-21 19:56:21 -07:00
Justine Tunney 38728cef79 Add poll() embedded webserver demo to redbean 2022-04-21 19:13:19 -07:00
Justine Tunney 0dca4c5799 Improve redbean ctrl-d handling 2022-04-21 16:17:48 -07:00
Justine Tunney c39d6111f2 Improve redbean ctrl-c handling 2022-04-21 15:24:44 -07:00
Justine Tunney 1599b818d9 Make more fixes and improvements 2022-04-21 13:44:59 -07:00
Gautham 01b25e267b Fix basename error with cp.com (#389) 2022-04-21 09:30:44 -07:00
Justine Tunney 9d61e23c80 Make more fixes and improvements
This change attempts to fix some report build issues. It also builds
upon development work described in previous changes.
2022-04-21 09:18:45 -07:00
Justine Tunney 9bfa6ec06e Add more documentation to redbean
This change also improves the unix module, adding a reboot() system call
for fun and profit, fixing the execve() api, and a printimage release.
2022-04-21 04:01:42 -07:00
Justine Tunney 87396f43bc Flatten InfoZIP directory and fix build issues 2022-04-20 22:40:33 -07:00
Justine Tunney ae638c0850 Fix bugs and make improvements
- Get clone() working on FreeBSD
- Increase some Python build quotas
- Add more atomic builtins to chibicc
- Fix ASAN poisoning of alloca() memory
- Make MODE= mandatory link path tinier
- Improve the examples folder a little bit
- Start working on some more resource limits
- Make the linenoise auto-complete UI as good as GNU readline
- Update compile.com, avoiding AVX codegen on non-AVX systems
- Make sure empty path to syscalls like opendir raises ENOENT
- Correctly polyfill ENOENT vs. ENOTDIR on the New Technology
- Port bestline's paredit features to //third_party/linenoise
- Remove workarounds for RHEL 5.0 bugs that were fixed in 5.1
2022-04-20 10:05:34 -07:00
Paul Kulchenko c3fb624647 Redbean fix text detect (#388)
This resets istext between requests, which may cause response to be
zipped when not needed (on non-text content).
2022-04-20 00:24:25 -07:00
Justine Tunney 5a132f9652 Add seccomp bpf sandboxing to redbean
It's now possible to pass the `-S` or `-SS` flags to sandbox redbean
worker proecsses after they've been forked. The first `-S` flag is
intended to be a permissive builtin policy that limits system calls to
only that which the various parts of redbean serving need. The second
`-SS` flag is intended to be more restrictive, preventing things like
the Lua extensions you download off the web from using the HTTP client
or sockets APIs. In upcoming changes you'll be able to implement your
own Berkeley Packet Filter sandbox programs and load them via Lua.
2022-04-18 08:54:42 -07:00
Justine Tunney 7166679620 Fix bugs and add security features to redbean
- Fix a regression with the previous change that broke redbean
- Add chroot(), resource limit, seccomp, and other stuff to redbean
- Write lots and lots of documentation
- Iron out more system call issues
2022-04-18 00:01:26 -07:00
Justine Tunney f1dfa4bdfa Add some more builtins to chibicc
https://justine.lol/cosmopolitan/documentation.html should now contain a
lot of functions that had been missing previously due to not having them
2022-04-17 13:07:07 -07:00
Justine Tunney ab38f0823d Make small fixes 2022-04-17 10:40:32 -07:00
Justine Tunney 80308079ec Disable redbean repl on windows 2022-04-16 23:52:03 -07:00
ProducerMatt 4a6a34db83 Makefile: incorrect executable location (#387) 2022-04-16 22:09:36 -07:00
Justine Tunney a6b02ce5a6 Add lua repl interface to redbean
You can now interact with the global web server state on the command
line, which the web server is running. This supports Emacs shortcuts
with history, readline parity, <tab> completions, plus hints. Enjoy!
2022-04-16 20:31:16 -07:00
Justine Tunney f6b6204b9e Add readline-style completions to Lua/Python/Almquist 2022-04-16 15:10:05 -07:00
Justine Tunney 933f33bcc1 Include variables in redbean lua traceback 2022-04-16 12:49:34 -07:00
Justine Tunney be7c5e1071 Fix debug mode build 2022-04-16 12:25:53 -07:00
Justine Tunney dc0ea6640e Fix bugs with recent change
This change makes further effort towards improving our poll()
implementation on the New Technology. The stdin worker didn't work out
so well for Python so it's not being used for now. System call tracing
with the --strace flag should now be less noisy now on Windows unless
you modify the strace.internal.h defines to turn on some optional ones
that are most useful for debugging the system call wrappers.
2022-04-16 10:40:23 -07:00
Justine Tunney 933411ba99 Improve synchronization
- Fix bugs in kDos2Errno definition
- malloc() should now be thread safe
- Fix bug in rollup.com header generator
- Fix open(O_APPEND) on the New Technology
- Fix select() on the New Technology and test it
- Work towards refactoring i/o for thread safety
- Socket reads and writes on NT now poll for signals
- Work towards i/o completion ports on the New Technology
- Make read() and write() intermittently check for signals
- Blinkenlights keyboard i/o so much better on NT w/ poll()
- You can now poll() files and sockets at the same time on NT
- Fix bug in appendr() that manifests with dlmalloc footers off
2022-04-15 15:31:55 -07:00
Paul Kulchenko 233144b19d Fix memory deallocation while yielding in redbean. (#384)
The yielded coroutine was removed from the stack too early,
leaving it not being anchored, which led to memory freed prematurely.
2022-04-14 18:13:53 -07:00
Justine Tunney fb7e8ef1e6 Add more raw system calls to redbean
We now have execve, setitimer, sigaction, sigsuspend, and sigprocmask.
2022-04-13 14:43:42 -07:00
Justine Tunney 281a0f2730 Implement raw system call for redbean lua code
You can now call functions like fork() from Lua and it'll work across
all supported platforms, including Windows. This gives you a level of
control of the system that Lua traditionally hasn't been able to have
due to its focus on old portable stdio rather modern POSIX APIs. Demo
code has been added to redbean-demo.com to show how it works.

This change also modifies Lua so that integer literals with a leading
zero will be interpreted as octal. That should help avoid shooting in
the foot with POSIX APIs that frequently use octal mode bits.

This change fixes a bug in opendir(".") on New Technology.

Lastly, redbean will now serve crash reports to private network IPs.
This is consistent with other frameworks. However that isn't served
to public IPs unless the -E flag is passed to redbean at startup.
2022-04-13 08:53:24 -07:00
Justine Tunney f684e348d4 Improve signals and memory protection
- Document sigaction()
- Simplify New Technology fork() code
- Testing and many bug fixes for mprotect()
- Distribute Intel Xed ILD in the amalgamation
- Turn Xed enums into defines to avoid DWARF bloat
- Improve polyfilling of SA_SIGINFO on BSDs and fix bugs
- setpgid(getpid(), getpid()) on Windows will ignore CTRL-C
- Work around issues relating to NT mappings being executable
- Permit automatic executable stack override via `ape_stack_pf`
2022-04-12 22:11:00 -07:00
Justine Tunney c95c9d9508 Trim down MODE= linkage slightly 2022-04-12 08:05:22 -07:00
Justine Tunney 046c7ebd4a Improve locks and signals
- Introduce fast spinlock API
- Double rand64() perf w/ spinlock
- Improve raise() on New Technology
- Support gettid() across platforms
- Implement SA_NODEFER on New Technology
- Move the lock intrinsics into LIBC_INTRIN
- Make SIGTRAP recoverable on New Technology
- Block SIGCHLD in wait4() on New Technology
- Add threading prototypes for XNU and FreeBSD
- Rewrite abort() fixing its minor bugs on XNU/NT
- Shave down a lot of the content in libc/bits/bits.h
- Let signal handlers modify CPU registers on New Technology
2022-04-12 05:20:17 -07:00
Justine Tunney f68f1789bd Add XNU magnums for preadv() and writev()
Apple introduced support sometime between 2018 and 2020.
2022-04-12 01:10:26 -07:00
Justine Tunney e500e8fb48 Delete some junk constants
Some of these are from userspace APIs and therefore don't need to be in
consts.sh. Others are poorly supported, rarely used, niche hardware not
in scope for support any time soon. Ideally, we should only have listed
the stuff that's supported really well. This is a conservative deletion
2022-04-11 23:49:53 -07:00
Justine Tunney a157940ba6 Delete most undocumented New Technology APIs
This change removes LIBC_KERNELBASE which is legacy code from the
initial import which was generated off a script that resolved the
delegated references, on Windows 10. All the important stuff here
should have long since been filed under kernel32.dll for windows7

Many FooA functions that were never assigned an arity are removed
because we almost never use the ASCII versions of WIN32 functions
therefore it's not worth having them slowing down the build. Some
other functions that overlap uncomfortably with libc are gone too

If something you need was removed, file an issue we'll restore it
2022-04-11 23:41:12 -07:00
Paul Kulchenko 183b3ed6a2 Update lsqlite3 to add update hook support for redbean (#383) 2022-04-09 12:36:44 -07:00
Paul Kulchenko 668dc42bac Reset Lua stack to drop unused returned values (#382)
Since some Lua code may return values that will be left on stack, they
need to be removed to avoid growing the stack unnecessarily.
2022-04-09 12:36:05 -07:00
Justine Tunney d31bebdd2d Add LISP interpreter 2022-04-07 20:30:22 -07:00
Justine Tunney 4f98ad1054 Improve system call support on NT
- Improve i/o perf on New Technology
- Code cleanup on read() for New Technology
- Fix bad bug with dup() of socket on New Technology
- Clean up some more strace errors on New Technology
2022-04-07 20:30:04 -07:00
Justine Tunney 29bf8b1a30 Make improvements
- Make rand64() thread safe
- Introduce lemur64 lcg prng
- Improve strace on New Technology
- Improve msync() on New Technology
2022-04-07 00:15:35 -07:00
Justine Tunney 43ba3009b2 Make another minor readme change 2022-04-06 13:07:51 -07:00
Justine Tunney 722d3df711 Delete old commented out Clang tests
See #343
2022-04-06 13:02:15 -07:00
Justine Tunney 837ff6cb49 Update readme to recommend nightlies 2022-04-06 12:55:14 -07:00
Justine Tunney 42fbcff149 Fix some more kinks in strace.com 2022-04-06 12:32:43 -07:00
Justine Tunney c13142dad2 Fix redbean certificate free error
This fixes a regression from a change a few weeks ago in git commit
af645fcbec which caused certificates to
not be free()'d correctly if the certificates are chained. dlmalloc()
should have printed an error in most build modes. ASAN caught it too.
2022-04-06 11:05:34 -07:00
Justine Tunney f59b5dbd7c Get strace.com working beautifully 2022-04-06 09:43:11 -07:00
Justine Tunney 82f2e758c7 Change the default command prompts 2022-04-06 06:56:11 -07:00
Justine Tunney ce9bdbb0bf Delete some more unneeded Make code 2022-04-06 06:55:08 -07:00
Justine Tunney 58d74ede4c Flatten GNU Make directory structure 2022-04-06 00:22:54 -07:00
Justine Tunney 04d39d47f1 Fix basename() and dirname() 2022-04-06 00:13:44 -07:00
Justine Tunney 9b11206ae3 Improve alloca() safety even more 2022-04-01 22:50:15 -07:00
Justine Tunney 38cb6e71ca Improve alloca() memory safety
Now that all the bugs have been wormed out of the ASAN memory module we
can successfully check for underruns on large stack allocations.
2022-04-01 22:44:43 -07:00
Justine Tunney 1ff1854107 Fix rusage memory reporting on NT
This issue was caused by c23b6ecc3 a few days ago, where we started
polling that information for the first time.
2022-03-31 17:07:30 -07:00
Sami Samhuri 27bc49034c Fix a couple typos (#361) 2022-03-29 22:54:09 -07:00
Sami Samhuri 1ccd269cdd Fix comment pointing at vfork.S (#362)
Looks like this comment was slightly out of date due to some file renaming.
2022-03-29 22:53:36 -07:00
Paul Kulchenko 2fe386ff8c Fix Redbean EncodeUrl params w/o values (#376) 2022-03-27 22:36:43 -07:00
Justine Tunney 072e1d2910 Make signal handling work well across platforms
- Fix sigsuspend() on XNU
- Fix strsignal() on non-Linux
- Add unit tests for strsignal()
- Add unit tests for setitimer()
- Add unit tests for sigsuspend()
- Rewrite setitimer() for New Technology
- Rewrite nanosleep() for New Technology
- Polyfill SIGALRM on the New Technology
- select(0,0,0,0) on NT now calls pause()
- Remove some NTDLL calls that aren't needed
- Polyfill SA_NOCLDWAIT on the New Technology
- Polyfill SA_RESETHAND on the New Technology
- Polyfill sigprocmask() on the New Technology
- Polyfill SIGCHLD+SIG_IGN on the New Technology
- Polyfill SA_RESTART masking on the New Technology
- Deliver console signals from main thread on New Technology
- Document SA_RESTART behavior w/ @sarestartable / @norestart
- System call trace in MODE=dbg now prints inherited FDs and signal mask
2022-03-25 07:28:57 -07:00
Justine Tunney 3b9e66ecba Add Linux ptrace() tutorial 2022-03-24 08:00:36 -07:00
Justine Tunney 98909b1391 Fix more things
- Update a couple unicode data files
- Disable strace during logger calls
- SQLite now uses pread() / pwrite()
- pread() past EOF on NT now returns 0
- Make the NT mmap() and fork() code elegant
- Give NT a big performance boost with memory
- Add many more mmap() tests to prove it works
2022-03-24 00:12:29 -07:00
Justine Tunney b90fa996b4 Get life.com in MODE=tiny back down to 12kb 2022-03-23 08:09:01 -07:00
Justine Tunney 23b72eb617 Add support for symbol table in .com files
This change fixes minor bugs and adds a feature, which lets us store the
ELF symbol table, inside the ZIP directory. We use the path /zip/.symtab
which can be safely removed using a zip editing tool, to make the binary
smaller after compilation. This supplements the existing method of using
a separate .com.dbg file, which is still supported. The intent is people
don't always know that it's a good idea to download the debug file. It's
not great having someone's first experience be a crash report, that only
has numbers rather than symbols. This will help fix that!
2022-03-23 06:34:46 -07:00
Justine Tunney 393ca4be40 Polyfill SIGCHLD on the New Technology
redbean will now cleanup child processes properly. New accounting
information is available too, such as page faults and memory usage. The
way it works is Cosmopolitan Libc samples the process collection on
entry into read() and poll() to see if SIGCHLD needs to be raised.

This change also fixes an issue with chibicc /tmp cleanup. There was
also a regression in MODE=dbg because STL needed ASAN runtime support.
2022-03-22 21:31:12 -07:00
Justine Tunney c23b6ecc31 Fix some win32 definitions
You can now use psapi.dll and pdh.dll. Some TODOs for Windows have been
cleared out. We might have a working load average for the platform that
should help GNU Make work well.
2022-03-22 19:54:36 -07:00
Justine Tunney e2e0b042c1 Delete redundant libcxx files 2022-03-22 13:05:00 -07:00
Justine Tunney 868af3f950 Import C++ Standard Template Library
You can now use the hardest fastest and most dangerous language there is
with Cosmopolitan. So far about 75% of LLVM libcxx has been added. A few
breaking changes needed to be made to help this go smoothly.

- Rename nothrow to dontthrow
- Rename nodiscard to dontdiscard
- Add some libm functions, e.g. lgamma, nan, etc.
- Change intmax_t from int128 to int64 like everything else
- Introduce %jjd formatting directive for int128_t
- Introduce strtoi128(), strtou128(), etc.
- Rename bsrmax() to bsr128()

Some of the templates that should be working currently are std::vector,
std::string, std::map, std::set, std::deque, etc.
2022-03-22 06:41:54 -07:00
Justine Tunney 5022f9e920 Trim down redbean a little bit
This makes redbean.c a little less long. It also reduces the size of
redbean-original.com from being 333K to 213K.
2022-03-21 19:32:30 -07:00
Paul Kulchenko 87029ac3f9 Redbean stream yield implementation (#370)
* Simplify handling of coroutine state
* Update redbean to allow yielding from Lua to support streaming
* Add stack checks for Lua resume calls in redbean
2022-03-21 18:07:30 -07:00
Justine Tunney e5314dedde Fix performance bottlenecks with nt fork redbean 2022-03-21 08:25:33 -07:00
Justine Tunney d57f87dc40 Fix some issues with zipos and redbean
- redbean.com -D /zip/dir/ now works, for pure fun
- possibly fixed bug with redbean serving empty files
- zipos stat() mode now indicates directories on windows

See #372
2022-03-21 07:36:36 -07:00
Justine Tunney 4881ae7527 Update Argon2 for style
- Make sure notice licenses are embedded
- Remove copyright and docs from headers
2022-03-21 07:27:03 -07:00
Nick Owens f78f2fcac3 redbean: allow symlinks for -D path (#372)
Previously ProgramDirectory called isdirectory which returns false for
symlinks. instead just check directory with stat, and allow user to pass
`-D /tmp/foo/`, where /tmp/foo might be a symlink.
2022-03-21 05:43:49 -07:00
Justine Tunney 5e8ae2d5bc Restart CI for New Technology and UBSAN hunting
Continuous Integration (via runit and runitd) is now re-enabled on win7
and win10. The `make test` command, which runs the tests on all systems
is now the fastest and most stable it's been since the project started.

UBSAN is now enabled in MODE=dbg in addition to ASAN. Many instances of
undefined behavior have been removed. Mostly things like passing a NULL
argument to memcpy(), which works fine with Cosmopolitan Libc, but that
doesn't prevents the compiler from being unhappy. There was an issue w/
GNU make where static analysis claims a sprintf() call can overflow. We
also now have nicer looking crash reports on Windows since uname should
now be supported and msys64 addr2line works reliably.
2022-03-21 04:32:57 -07:00
Paul Kulchenko d5ff2c3fb9 Make improvements to redbean (#373)
* Allow ProgramUniprocess to be set on Windows
* Add closing p tag to Maxmind example
* Enable using symlinked directories with redbean -A flag
* Rename gc in redbean for consistency
* Improve StoreFile by adding checks and skipping ./ in stored path
* Add freeing memory in redbean LuaEncodeUrl

Ref #366
2022-03-21 03:20:09 -07:00
Justine Tunney 0cb6b6ff4b Get Redbean fork() working on the New Technology
Now that we have understandable system call tracing on Windows, this
change rewrites many of the polyfill internals for that platform, to
help things get closer to tip top shape. Support for complex forking
scenarios had been in a regressed state for quite some time. Now, it
works! Subsequent changes should be able to address the performance.
2022-03-20 08:01:14 -07:00
Paul Kulchenko efedef6e65 Fix Lua EncodeUrl without values (#368) 2022-03-19 18:00:15 -07:00
Justine Tunney 39688a73e4 Polish recent changes and make improvements
- Simulate SIGPIPE on Windows NT
- Fix commandv() regression on Windows NT
- Fix sigprocmask() strace bug on OpenBSD
- Add many more system calls to --strace logging
- Make errno state more pristine in redbean strace
2022-03-19 03:37:00 -07:00
Justine Tunney 10a766ebd0 Delete VAX/OS2/DOS/AMIGA code from GNU Make 2022-03-18 21:39:44 -07:00
Justine Tunney 14e192e5ba Introduce --strace flag for system call tracing
This is similar to the --ftrace (c function call trace) flag, except
it's less noisy since it only logs system calls to stderr. Having this
flag is valuable because (1) system call tracing tells us a lot about
the behavior of complex programs and (2) it's usually very hard to get
system call tracing on various operating systems, e.g. strace, ktrace,
dtruss, truss, nttrace, etc. Especially on Apple platforms where even
with the special boot trick, debuggers still aren't guaranteed to work.

    make -j8 o//examples
    o//examples/hello.com --strace

This is enabled by default in MODE=, MODE=opt, and MODE=dbg. In MODE=dbg
extra information will be printed.

    make -j8 MODE=dbg o/dbg/examples
    o/dbg/examples/hello.com --strace |& less

This change also changes:

- Rename IsText() → _istext()
- Rename IsUtf8() → _isutf8()
- Fix madvise() on Windows NT
- Fix empty string case of inet_ntop()
- vfork() wrapper now saves and restores errno
- Update xsigaction() to yoink syscall support
2022-03-18 18:07:28 -07:00
Justine Tunney c541225af0 Add maxmind demo to redbean-demo.com 2022-03-18 15:31:54 -07:00
Justine Tunney c4d8f40422 Fix regression in kilo example 2022-03-18 14:30:45 -07:00
Justine Tunney 96bdd7e90f Update tzdata 2022-03-18 13:24:00 -07:00
Justine Tunney 390e22a8c6 Add ASAN guards to some asm data structures 2022-03-18 13:02:57 -07:00
Justine Tunney 6c2fd9ecc6 Clean old .source directive out of asm code 2022-03-18 12:43:21 -07:00
Justine Tunney b149a9bcc4 Minor fixes for image printing programs 2022-03-18 04:20:10 -07:00
Justine Tunney 9d2c8798c1 Fix backtrace test failure in MODE=rel 2022-03-18 03:48:37 -07:00
Justine Tunney d07a58a763 Fix inconsistency in SectorLambda license
SectorLambda is intended to be licensed ISC. SectorLambda is not
intended to be licensed under the TeX license.
2022-03-18 03:25:32 -07:00
Justine Tunney c371db6663 Add maxmind to redbean 2022-03-18 03:17:08 -07:00
Justine Tunney af645fcbec Make exciting improvements
- Add Lua backtraces to redbean!
- Wipe serving keys after redbean forks
- Audit redbean to remove free via exit
- Log SSL client ciphersuite preferences
- Increase ASAN malloc() backtrace depth
- Make GetSslRoots() behave as a singleton
- Move leaks.c from LIBC_TESTLIB to LIBC_LOG
- Add undocumented %n to printf() for newlines
- Fix redbean memory leak reindexing inode change
- Fix redbean memory leak with Fetch() DNS object
- Restore original environ after __cxa_finalize()
- Make backtrace always work after __cxa_finalize()
- Introduce COUNTEXPR() diagnostic / benchmark tool
- Fix a few more instances of errno being clobbered
- Consolidate the ANSI color disabling internal APIs
2022-03-18 03:02:00 -07:00
Justine Tunney f5831a62fa Add SectorLambda 2022-03-17 14:12:41 -07:00
Justine Tunney 741c836e9d Fix build flags for kprintf() 2022-03-17 13:54:16 -07:00
Justine Tunney 46a3b88594 Have ASAN errors show origin of memory 2022-03-17 00:53:45 -07:00
Justine Tunney e20fcf02c1 Fix redbean build flake 2022-03-16 17:37:24 -07:00
Justine Tunney bf62140377 Choose better deterministic timestamp for zip 2022-03-16 16:59:45 -07:00
Justine Tunney b45d50b690 Make improvements
- Fix build flakes
- Polyfill SIGWINCH on Windows
- Fix an execve issue on Windows
- Make strerror show more information
- Improve cmd.exe setup/teardown on Windows
- Support bracketed paste mode in Blinkenlights
- Show keyboard shortcuts in Blinkenlights status bar
- Fixed copy_file_range() and copyfile() w/ zip filesystem
- Size optimize GetDosArgv() to keep life.com 12kb in size
- Improve Blinkenlights ability to load weird ELF executables
- Fix program_executable_name and add GetInterpreterExecutableName
- Make Python in tiny mode fail better if docstrings are requested
- Update Python test exclusions in tiny* modes such as tinylinux
- Add bulletproof unbreakable kprintf() troubleshooting function
- Remove "oldskool" keyword from ape.S for virus scanners
- Fix issue that caused backtraces to not print sometimes
- Improve Blinkenlights serial uart character i/o
- Make clock_gettime() not clobber errno on xnu
- Improve sha256 cpuid check for old computers
- Integrate some bestline linenoise fixes
- Show runit process names better in htop
- Remove SIGPIPE from ShowCrashReports()
- Make realpath() not clobber errno
- Avoid attaching GDB on non-Linux
- Improve img.com example
2022-03-16 13:40:10 -07:00
Paul Kulchenko 2a938b3eaa Use last X-Forwarded-For header (#367)
This header is non-standard but AWS seems to need this.
2022-03-14 17:21:15 -07:00
Paul Kulchenko cfc557f7c7 Add storing folders in redbean from CLI (#366) 2022-03-14 17:19:31 -07:00
Paul Kulchenko 38112aeb20 Fix Redbean when file is read-only (#365) 2022-03-14 17:13:28 -07:00
Paul Kulchenko 22409b2b5e Redbean SSL identification (#360)
* Let Fetch() be used earlier in initialization
* Have ssl log messages show cert name
* Introduce GetSslIdentity Lua API
2022-03-14 17:11:05 -07:00
Paul Kulchenko abac6f729c Add ProgramUniprocess to redbean (#364) 2022-03-07 18:15:44 -08:00
Paul Kulchenko 1e3c5e10ad Update docs on chmod permissions (#336)
Closes #335
2022-03-07 18:13:49 -08:00
Paul Kulchenko 4abae20172 Redbean StoreAsset fix and lua cli (#326)
* Fix StoreAsset update for existing assets in redbean
* Add lua code execution and asset storage from redbean command line
2022-03-04 18:47:15 -08:00
Paul Kulchenko 9bab356c37 Fix double Content-Type after SetHeader (#327) 2022-03-04 18:44:39 -08:00
Paul Kulchenko 0f035119b1 Fix exec example (#341) 2022-03-04 17:51:32 -08:00
Paul Kulchenko db6152e21d Fix executable name generation under QEMU (#347)
Closes #346
2022-03-04 17:49:43 -08:00
Paul Kulchenko d938b89f4f Fix redbean browser launch before server starts (#359) 2022-03-04 17:41:19 -08:00
Justine Tunney d6a039821f Release redbean 1.5 2021-11-15 07:39:38 -08:00
Oguz Bektas f12c6c1b5e Fix comment for nc example (#323)
The "Host" header is missing from the command for making the
request, so we get empty response.
2021-11-15 06:48:46 -08:00
Paul Kulchenko fdb543cbb3 Fix incorrect reference in Lua encoding (#324) 2021-11-13 13:10:09 -08:00
Paul Kulchenko 1d6216a775 Add encode json and encode Lua functions to redbean (#322) 2021-11-13 12:49:29 -08:00
Jørgen Kvalsvik 206091617a Don't pass unused args to printf (#311)
The final print does not print any content, only the byte-offset of the
end-block, which makes the A, B params unused. From gcc:

    bd.c:77:17: warning: too many arguments for format [-Wformat-extra-args]
        if (o) printf("%08x\n", o, A, B);
2021-11-13 01:10:45 -08:00
Justine Tunney 777d08a839 Fix mistake with IsHeaderRepeatable 2021-11-12 20:54:30 -08:00
Justine Tunney 98192c1079 Add undocumented IsHeaderRepeatable to redbean
This API is intended to help unblock framework code. It should
ideally be abstracted by redbean which is why it's undocumented.

See #97
2021-11-12 19:03:35 -08:00
Justine Tunney 7064d736e3 Increase some quotas
- Double mem quota (fixes #296) because linking Python is
  expensive and not easily tuned on a case-by-case basis

- Increase latency greatly for mkdeps tool since it's the
  first thing that runs and effetively manages to load
  17,000 files into the hard disk cache (see #97)
2021-11-12 16:38:54 -08:00
Justine Tunney 245a9c206b Fix redbean build
Also ran clang-format on redbean.c

See #312 and #321
2021-11-12 16:34:18 -08:00
Paul Kulchenko 1bdc8faa65 Add redbean function for simple HMAC (#321) 2021-11-12 16:26:14 -08:00
Paul Kulchenko 8f05990d5a Extend GetZipPaths to accept an optional prefix (#320) 2021-11-12 15:28:05 -08:00
Gautham 7fe9e70117 Backporting METH_FASTCALL from Python 3.7 (#317)
* dict copy speedup

refer to bpo-31179 or python/cpython@boa7a037b8fde

* __build_class__() uses METH_FASTCALL

refer python/cpython@69de71b255
refer python/cpython@773dc6dd06

a single test related to __prepare__ fails.

* type_prepare uses METH_FASTCALL

refer python/cpython@d526cfe546
refer python/cpython@80ab22fa2c

the prepare-related test still fails.  It's just related to the error
message format though.

* separate into ParseStack and ParseStackAndKeywords

refer python/cpython@6518a93cb1
refer python/cpython@3e1fad6913
refer python/cpython@c0083fc47d

* Add _PyArg_NoStackKeywords

refer python/cpython@29d39cc8f5

* _PyStack_UnpackDict now returns int

refer python/cpython@998c20962c

* METH_FASTCALL changes to .inc files

done via python's Argument Clinic tool,
refer python/cpython@259f0e4437

* Added _PyArg_UnpackStack

refer python/cpython@fe54dda08

* Argument Clinic FASTCALL again

refer python/cpython@0c4a828ca

* Argument Clinic for ordered dictionary object

refer python/cpython@b05cbac052

* speed up getargs

refer python/cpython@1741441649

* FASTCALL for sorted, next, and getattr

refer python/cpython@5a60ecaa7a
refer python/cpython@fda6d0acf0
refer python/cpython@84b388bb80

* Optimize methoddescr_call

refer python/cpython@2a1b676d1f
refer python/cpython@c52572319c
refer python/cpython@35ecebe165
refer python/cpython@8128d5a491

* cleanup _PyMethodDef_RawFastCallDict

refer python/cpython@0a2e46835d
refer python/cpython@98ccba8344
refer python/cpython@c89ef828cf
refer python/cpython@250e4b0063

* print now uses METH_FASTCALL

refer python/cpython@c3858bd7c6
refer python/cpython@bd584f169f
refer python/cpython@06d34393c2

* _struct module now uses Argument Clinic

refer python/cpython@3f2d10132d

* make deque methods faster

refer python/cpython@dd407d5006

* recursive calls in PyObject_Call

refer python/cpython@7399a05965

only partially ported, because RawFastCallKeywords hasn't been ported

* add macros

refer python/cpython@68a001dd59

* all tests pass in MODE=dbg

* convert some internal functions to FASTCALL

__import__ might need to be changed later, if it is possible to backport
the METH_FASTCALL | METH_KEYWORDS flag distinction later.

* speed up unpickling

refer python/cpython@bee09aecc2

* added _PyMethodDef_RawFastCallKeywords

refer python/cpython@7399a05965

* PyCFunction_Call performance

refer python/cpython@12c5838dae

* avoid PyMethodObject in slots

main change in python/cpython@516b98161a
test_exceptions changed in python/cpython@331bbe6aaa
type_settattro changed in python/cpython@193f7e094f
_PyObject_CallFunctionVa changed in python/cpython@fe4ff83049

* fix refcount error found in MODE=dbg

all tests now pass in MODE=dbg
2021-11-12 15:26:57 -08:00
Gautham 6f658f058b Change noinline to dontinline (#312)
We defined `noinline` as an abbreviation for the longer version
`__attribute__((__noinline__))` which caused name clashes since
third party codebases often write it as `__attribute__((noinline))`.
2021-11-12 15:12:18 -08:00
Paul Kulchenko ca611efc43 Redbean getstatus and more (#308)
* Add GetBody() Lua API to redbean.
   This improves consistency with RFC 7230 terminology and
   should be favored over the old GetPayload function.
* Add GetStatus() API to redbean.
   This is useful to get status after it's changed/set by Redbean,
   for example if 505 or 508 is set when ServeRedirect is called.
* Introduce GetAssetComment() API to redbean.
   This function should be favored over the old name GetComment().
* Introduce IsLoopbackClient() API to redbean
* Limit redbean reason to 128 chars when set instead of reporting an error
2021-11-12 15:00:41 -08:00
Paul Kulchenko e5d1536256 Redbean doc updates (#307)
* Fix redbean re.NEWLINE documentation
* Add documentation for method and body parameters in redbean Fetch
* Add documentation for redbean ProgramAddr
* Update redbean SetHeader documentation to clarify behavior with Serve* calls
2021-11-01 18:52:02 -07:00
Gautham d7ff346b52 Add some Python 3.7 backports (#306)
* make.com now uses stack size of 2mb
* optimize _PyCFunction_FastCallKeywords
* backport python@cpython/7fc252adfbedece75f2330bcfdadbf84dee7836f
2021-10-29 22:54:14 -07:00
Justine Tunney 903cc38c37 Revert "Make MODE=tiny not depend on default build"
This reverts commit 30cd28b1f8.
2021-10-26 17:14:38 -07:00
ahgamut 3616f2a432 get make to build 2021-10-26 14:58:05 -07:00
ahgamut 7a50dba812 changes made to enable compilation 2021-10-26 14:58:05 -07:00
ahgamut 19f70a154e add make-4.3.tar.gz 2021-10-26 14:58:05 -07:00
jacereda 0a0997a872 Add WIN32 symbols for OpenGL support (#304) 2021-10-26 14:18:13 -07:00
Justine Tunney 30cd28b1f8 Make MODE=tiny not depend on default build
This issue was spotted by @ahgamut in #292.
2021-10-25 16:26:22 -07:00
Justine Tunney 91d783352a Apply fixup to cthread initialization (#301)
Cosmopolitan Threads are currently Linux-only (with some NetBSD
and Windows support too!). This change ensures we only initialize
the high-level threading runtime when Cosmopolitan Threads are used.
2021-10-25 16:22:10 -07:00
Florian Lemaitre 45a7435788 [WIP] Threading phase 2 (#301)
* Exponential back-off
* Removed "native" specifier
* Abstract away Futex for cthread
* Complete setup for TLS (including main thread)
2021-10-25 16:02:26 -07:00
Paul Kulchenko 660ff56d40 Redbean fix lua panic (#300)
* Fix Lua panic in redbean when calling request/connection functions from .init.lua

The error calls were triggered appropriately, but used the global Lua
state instead of the current Lua state (within protected call), which
triggered Lua panic. This executes the error in the proper context.

* Rename global Lua state object to make its (mis-)usage easier to detect
2021-10-25 14:54:56 -07:00
Paul Kulchenko c00755f921 Improve error messages for consistency (#297) 2021-10-25 14:44:04 -07:00
Paul Kulchenko 013f03e33f Redbean fix redirect (#294)
* Fix redbean crash during redirect in debug logging mode

* Add reset for redirect loop check in redbean Route

Without this reset a combination of RoutePath() and Route() calls
could return "508 loop detected", since RoutePath could be called
twice for the same redirected path.

The protection against looping is still there, as it can only
loop inside the Route() call (as it always serves something).

* Update redbean redirect message for clarity
2021-10-25 14:04:57 -07:00
Gautham 49db877fbe Minimize Python startup imports (#292)
* get_exports_list should return list
* remove unintentional `CC=clang` in makefile
* avoid importing sysconfig during startup

site.py requires only a couple of functions from sysconfig, but needs to
load the entirety of sysconfig to get those functions. This commit
makes it such that sysconfig is imported only when sys.platform is darwin.

* remove redundant constants from stat module

The constants are only there in case the C implementation (ie the _stat
module) is not available. With Cosmopolitan the _stat module is always
available. The entire Lib/stat.py file can be removed if the Windows-based
constants can be moved into the Modules/_stat.c.

* minimal changes to os.py

python checks os-based assumptions at startup, some of  which can be
bypassed since this is Cosmopolitan Python.
2021-10-25 14:04:04 -07:00
Et7f3 253ac31a64 Fix build from docker build with alpine from macOS (#286)
* fix(build): Remove unused variable and fix const warning.
* fix(build): Fix others -Werror=discarded-qualifiers
Co-authored-by: Élie BRAMI <[email protected]>
2021-10-25 13:39:05 -07:00
Justine Tunney 67b5200a0b Add MODE=optlinux build mode (#141) 2021-10-14 19:36:49 -07:00
Justine Tunney 226aaf3547 Improve memory safety
This commit makes numerous refinements to cosmopolitan memory handling.

The default stack size has been reduced from 2mb to 128kb. A new macro
is now provided so you can easily reconfigure the stack size to be any
value you want. Work around the breaking change by adding to your main:

    STATIC_STACK_SIZE(0x00200000);  // 2mb stack

If you're not sure how much stack you need, then you can use:

    STATIC_YOINK("stack_usage_logging");

After which you can `sort -nr o/$MODE/stack.log`. Based on the unit test
suite, nothing in the Cosmopolitan repository (except for Python) needs
a stack size greater than 30kb. There are also new macros for detecting
the size and address of the stack at runtime, e.g. GetStackAddr(). We
also now support sigaltstack() so if you want to see nice looking crash
reports whenever a stack overflow happens, you can put this in main():

    ShowCrashReports();

Under `make MODE=dbg` and `make MODE=asan` the unit testing framework
will now automatically print backtraces of memory allocations when
things like memory leaks happen. Bugs are now fixed in ASAN global
variable overrun detection. The memtrack and asan runtimes also handle
edge cases now. The new tools helped to identify a few memory leaks,
which are fixed by this change.

This change should fix an issue reported in #288 with ARG_MAX limits.
Fixing this doubled the performance of MKDEPS.COM and AR.COM yet again.
2021-10-13 17:27:13 -07:00
Florian Lemaitre a0b39f886c [WIP] Threading (#282)
* Thread creation
* Proper thread creation and exit
* Join/Detach protocol
* Added semaphore with futex (hopefully fast)
2021-10-13 11:26:05 -07:00
Gautham d852640a1e Add Python ftrace contextmanager (#285) 2021-10-13 11:00:25 -07:00
Justine Tunney 425a57080d Fix minor issues 2021-10-08 20:51:11 -07:00
Justine Tunney bba144246a Improve Das Blinkenlights
It's now possible to scroll panels is the TUI while the display
is blocked on input. INT 16h now translates UTF-8 to CP-437 and
displays unmappable characters using a lambda symbol. Bracketed
paste mode guards will also be filtered out.
2021-10-08 16:44:19 -07:00
Justine Tunney 7061c79c22 Make fixes, improvements, and chibicc python bindings
- python now mixes audio 10x faster
- python octal notation is restored
- chibicc now builds code 3x faster
- chibicc now has help documentation
- chibicc can now generate basic python bindings
- linenoise now supports some paredit-like features

See #141
2021-10-08 08:41:57 -07:00
Justine Tunney 28997f3acb Make mkdeps.com go faster
This program usually runs once at the begininng of each GNU Make
invocation. It generates an o//depend file with 170,000 lines of
Makefile code to define source -> headers relationships.

This change makes that take 650 milliseconds rather than 1,100ms
by improving the performance of strstr(), using longsort(), plus
migrating to the new append library.
2021-10-04 06:46:46 -07:00
Justine Tunney 725f4d79f6 Apply fixes and speedups 2021-10-04 03:23:31 -07:00
Justine Tunney 7521bf9e73 Add stack overflow checking to Python 2021-10-02 10:50:41 -07:00
Justine Tunney 47a53e143b Productionize new APE loader and more
The APE_NO_MODIFY_SELF loader payload has been moved out of the examples
folder and improved so that it works on BSD systems, and permits general
elf program headers. This brings its quality up enough that it should be
acceptable to use by default for many programs, e.g. Python, Lua, SQLite
and Python. It's the responsibility of the user to define an appropriate
TMPDIR if /tmp is considered an adversarial environment. Mac OS shall be
supported by APE_NO_MODIFY_SELF soon.

Fixes and improvements have been made to program_executable_name as it's
now the one true way to get the absolute path of the executing image.

This change fixes a memory leak in linenoise history loading, introduced
by performance optimizations in 51904e2687
This change fixes a longstanding regression with Mach system calls, that
23ae9dfceb back in February which impacted
our sched_yield() implementation, which is why no one noticed until now.

The Blinkenlights PC emulator has been improved. We now fix rendering on
XNU and BSD by not making the assumption that the kernel terminal driver
understands UTF8 since that seems to break its internal modeling of \r\n
which is now being addressed by using \e[𝑦H instead. The paneling is now
more compact in real mode so you won't need to make your font as tiny if
you're only emulating an 8086 program. The CLMUL ISA is now emulated too

This change also makes improvement to time. CLOCK_MONOTONIC now does the
right thing on Windows NT. The nanosecond time module functions added in
Python 3.7 have been backported.

This change doubles the performance of Argon2 password stretching simply
by not using its copy_block and xor_block helper functions, as they were
trivial to inline thus resulting in us needing to iterate over each 1024
byte block four fewer times.

This change makes code size improvements. _PyUnicode_ToNumeric() was 64k
in size and now it's 10k. The CJK codec lookup tables now use lazy delta
zigzag deflate (δzd) encoding which reduces their size from 600k to 200k
plus the code bloat caused by macro abuse in _decimal.c is now addressed
so our fully-loaded statically-linked hermetically-sealed Python virtual
interpreter container is now 9.4 megs in the default build mode and 5.5m
in MODE=tiny which leaves plenty of room for chibicc.

The pydoc web server now accommodates the use case of people who work by
SSH'ing into a different machine w/ python.com -m pydoc -p8080 -h0.0.0.0

Finally Python Capsulae delenda est and won't be supported in the future
2021-10-02 08:27:03 -07:00
Justine Tunney 9cb54218ab Add error checks to Python objectifier (#281)
PYOBJ.COM was failing when statically analyzing _pyio.py in MODE=dbg
because co_consts contained a big number, which dirtied the interpreter
exception state. We now do comprehensive error checking w/ Python API.

The -DSTACK_FRAME_UNLIMITED CPPFLAG has been removed from DES since its
self test function has been fixed to use heap memory rather than making
aggressive use of the stack.

This change also fixes a regression with function tracing (the --ftrace
flag a.k.a. ftrace_install() a.k.a. cosmo.ftrace) in ASAN build modes.
Lastly, the _tracemalloc module should now always be available for use
in MODE=dbg.
2021-10-02 06:17:17 -07:00
Gautham 57f0eed382 Fix Pyston speedups (#281)
We remove (i.e. hide behind a debug ifdef) the recursion checking methods,
and the memory hooks and memory allocator methods. ASAN mode has no
PYMALLOC, so we need a macro. Fix build break with des.c stack allocation.
2021-10-02 01:28:51 -07:00
Gautham 2fe8571010 Add zipimport hook at the end just in case (#280)
In Python, the zipimport path hook is usually the first entry in
sys.path_hooks, so that any zip files in sys.path can be handled
correctly. In the APE, the zipimport hook was removed because it was
relatively slow compared to Cosmopolitan when it came to handling
imports from the APE's internal zip store.

However, some python scripts (for example when pip installs some
packages) modify sys.path to consider a local zip file, and then attempt
to import from it. This change prevents potential "unable to import"
errors in such cases, so that Actually Portable Python can be more of a
drop-in improved replacement.
2021-09-29 01:50:34 -07:00
Paul Kulchenko 9cabdaabfd Add clock() polyfill on Windows (#279)
We normally use CLOCK_PROCESS_CPUTIME_ID but there's no clock_gettime()
on Windows NT.
2021-09-28 02:14:24 -07:00
Justine Tunney 39bf41f4eb Make numerous improvements
- Python static hello world now 1.8mb
- Python static fully loaded now 10mb
- Python HTTPS client now uses MbedTLS
- Python REPL now completes import stmts
- Increase stack size for Python for now
- Begin synthesizing posixpath and ntpath
- Restore Python \N{UNICODE NAME} support
- Restore Python NFKD symbol normalization
- Add optimized code path for Intel SHA-NI
- Get more Python unit tests passing faster
- Get Python help() pagination working on NT
- Python hashlib now supports MbedTLS PBKDF2
- Make memcpy/memmove/memcmp/bcmp/etc. faster
- Add Mersenne Twister and Vigna to LIBC_RAND
- Provide privileged __printf() for error code
- Fix zipos opendir() so that it reports ENOTDIR
- Add basic chmod() implementation for Windows NT
- Add Cosmo's best functions to Python cosmo module
- Pin function trace indent depth to that of caller
- Show memory diagram on invalid access in MODE=dbg
- Differentiate stack overflow on crash in MODE=dbg
- Add stb_truetype and tools for analyzing font files
- Upgrade to UNICODE 13 and reduce its binary footprint
- COMPILE.COM now logs resource usage of build commands
- Start implementing basic poll() support on bare metal
- Set getauxval(AT_EXECFN) to GetModuleFileName() on NT
- Add descriptions to strerror() in non-TINY build modes
- Add COUNTBRANCH() macro to help with micro-optimizations
- Make error / backtrace / asan / memory code more unbreakable
- Add fast perfect C implementation of μ-Law and a-Law audio codecs
- Make strtol() functions consistent with other libc implementations
- Improve Linenoise implementation (see also github.com/jart/bestline)
- COMPILE.COM now suppresses stdout/stderr of successful build commands
2021-09-28 01:52:34 -07:00
Paul Kulchenko fa7b4f5bd1 Updated lua-Argon2 to use Argon2_id variant
As recommended in RFC9106:
https://www.rfc-editor.org/rfc/rfc9106.html#name-recommendations
2021-09-27 21:42:40 -07:00
Paul Kulchenko b883473a6c Remove optimized blamka rounds from Argon2
Optimized blamka rounds occasionally SIGSERV for me,
so the reference implementation is used instead.
2021-09-27 21:42:40 -07:00
Paul Kulchenko af1153e057 Reorganize handling of added SSE intrinsics to simplify. 2021-09-27 21:42:40 -07:00
Paul Kulchenko 2e4b40042a Add missing Intel SSE2 intrinsics needed for Argon2 2021-09-27 21:42:40 -07:00
Paul Kulchenko 42aafc847e Update lua-argon2 for cosmopolitan/redbean 2021-09-27 21:42:40 -07:00
Paul Kulchenko b51db9f923 Add lua-argon2 module for redbean/lua 2021-09-27 21:42:40 -07:00
Paul Kulchenko 6fba9601ff Update Argon2 for cosmopolitan libc 2021-09-27 21:42:40 -07:00
Paul Kulchenko 9e7773a076 Add Argon2 implementation 2021-09-27 21:42:40 -07:00
jacereda 48a2668239 Fix bad dependency (#276) 2021-09-27 20:39:38 -07:00
Paul Kulchenko 5ffeaa8e33 Fix SSE scalar ops (#271) 2021-09-17 16:49:53 -07:00
Florian Lemaitre 18ccfeb919 Embed APE loader inside APE (#267)
The `ape-no-modify-self.o` bootloader now has an mmap-based
payload that helps read-only APE binaries be load faster since it
doesn't need to copy any files.
2021-09-17 16:49:10 -07:00
Justine Tunney b5f743cdc3 Begin incorporating Python unit tests into build
We now build a separate APE binary for each test so they can run in
parallel. We've got 148 tests running fast and stable so far.
2021-09-12 21:04:44 -07:00
Justine Tunney 51904e2687 Improve Python and Linenoise
This change reinvents all the GNU Readline features I discovered that I
couldn't live without, e.g. UTF-8, CTRL-R search and CTRL-Y yanking. It
now feels just as good in terms of user interface from the subconscious
workflow perspective. It's real nice to finally have an embeddable line
reader that's actually good with a 30 kb footprint and a bsd-2 license.

This change adds a directory to the examples folder, explaining how the
new Python compiler may be used.  Some of the bugs with Python binaries
have been addressed but overall it's still a work in progress.
2021-09-11 22:30:37 -07:00
Justine Tunney ad52387b74 Fix race condition in quickjs build config (#251)
qjsc.com now has a -n do nothing flag so the makefile can create a
localized binary. See also #267 where we have an exciting new change
aiming to address this particular APE gotcha.
2021-09-11 22:16:17 -07:00
agreppin 6ad0602392 quickjs: add repl + qjscalc (#251)
- Compile repl.js & qjscalc.js and include them in qjs.com zip
- Change quickjs-libc.c / js_os_poll to handle Windows console

This last change is needed because poll and select on Windows forbid
non-socket handles and Cosmopolitan Libc hasn't polyfilled that yet.
2021-09-11 16:52:49 -07:00
Paul Kulchenko 1b93066883 Add GetCookie to redbean Lua (#269) 2021-09-11 16:46:21 -07:00
Justine Tunney 559b024e1d Decentralize Python native module linkage
We can now link even smaller Python binaries. For example, the hello.com
program in the Python build directory is a compiled linked executable of
hello.py which just prints hello world. Using decentralized sections, we
can make that binary 1.9mb in size (noting that python.com is 6.3 megs!)

This works for nontrivial programs too. For example, say we want an APE
binary that's equivalent to python.com -m http.server. Our makefile now
builds such a binary using the new launcher and it's only 3.2mb in size
since Python sources get turned into ELF objects, which tell our linker
that we need things like native hashing algorithm code.
2021-09-07 11:40:11 -07:00
Justine Tunney dfa0359b50 Exclude .py files in MODE=rel / tiny 2021-09-06 19:34:57 -07:00
Justine Tunney 4f41f2184d Improve Python tree-shaking 2021-09-06 19:24:10 -07:00
Justine Tunney 5bb2275788 Get bzip2 to build 2021-09-06 17:36:47 -07:00
Justine Tunney a8fb68af4b Add bzip2 1.0.8 2021-09-06 15:51:57 -07:00
Paul Kulchenko 1eed7d47bd Add error reporting to redbean Slurp (#268)
This should allow `content = assert(Slurp(filename))` to work as
expected and report an error if file doesn't exist or can't be read.
2021-09-06 08:21:37 -07:00
Justine Tunney 44c87b83ff Implement tree-shaking for Python sources 2021-09-05 01:20:03 -07:00
Justine Tunney 81287b7ec0 Introduce Python objectifier (#259) 2021-09-04 15:44:00 -07:00
Justine Tunney 34b68f1945 Make mappings unlimited on NT
This change might also fix fork() in certain cases on NT.
2021-09-04 13:20:47 -07:00
Justine Tunney ab64c746cc Fix issues with InfoZip build
Artifacts weren't being invalidated because THIRD_PARTY_ZIP_SRCS_C
needed to be renamed THIRD_PARTY_ZIP_SRCS. This caused a build break.
2021-09-04 06:50:57 -07:00
Justine Tunney a81192e0b9 Fix some build breaks 2021-09-04 02:29:57 -07:00
Gautham 27f7ffd4fd Add speedups from pyston (#264)
This should make Python go 30% faster. It does that by trading
away some debuggability, like _tracemalloc. It can be re-enabled
using `make MODE=dbg`.
2021-09-04 02:21:37 -07:00
Paul Kulchenko 31dd714081 Add SetCookie method to redbean Lua (#265) 2021-09-04 02:12:12 -07:00
Justine Tunney 969174e155 Add APE interpreter example (#263) 2021-09-04 00:58:21 -07:00
Justine Tunney 5b60e5a37d Fix termios struct on Linux
The termios::c_cc field turned out to be incorrectly defined on Linux
due to some confusion between the glibc and kernel definitions. We'll
be using the kernel definition, since it has the strongest consensus.

Fields have been have been added to struct stat for BSD compatibility
such as st_birthtim, plus the GLIBC compatibility of isystem/sys/stat
has been improved.
2021-09-03 22:19:41 -07:00
Paul Kulchenko 0584684a82 Improve logger API (#262)
This breaking change improves naming consistency.

- Rename LOGF to INFOF
- Rename recently introduced ANYF to LOGF
- Remove V* log calls, as they are not being used
2021-09-03 21:14:26 -07:00
Justine Tunney 50937be752 Fix select() on Windows for timeout (#141) 2021-08-26 15:59:55 -07:00
Justine Tunney 3085ac7837 Improve system call support 2021-08-25 21:36:17 -07:00
Gautham 63b867bd2f Added _multiprocessing to Python (#259)
Also changed some PYTHON_YOINKs so that http.server would work in
MODE=tiny.
2021-08-25 19:45:59 -07:00
Paul Kulchenko 0b7737b20a Fix redbean StoreAsset w/ Deflate (#258) 2021-08-25 05:43:26 -07:00
Paul Kulchenko 818926dbae Fix redbean latency and resource usage logging (#257) 2021-08-22 20:16:10 -07:00
Justine Tunney 7d25fb0090 Import some Lua documentation
I personally find it easier to read the documentation in Emacs
using JavaDoc style comments.
2021-08-22 15:03:04 -07:00
Paul Kulchenko 41b9eb6873 Redbean log improvements (#256)
* Update log level for redbean messages for better log content
* Add categories to redbean log messages
* Reorganize Lua error logging for consistency
* Replace perror logging with FATALF
* Update server failure reporting to better identify response code
* Replace `flogf` with DIEF logging for consistency
2021-08-22 15:01:52 -07:00
Justine Tunney 00611e9b06 Improve ZIP filesystem and change its prefix
The ZIP filesystem has a breaking change. You now need to use /zip/ to
open() / opendir() / etc. assets within the ZIP structure of your APE
binary, instead of the previous convention of using zip: or zip! URIs.
This is needed because Python likes to use absolute paths, and having
ZIP paths encoded like URIs simply broke too many things.

Many more system calls have been updated to be able to operate on ZIP
files and file descriptors. In particular fcntl() and ioctl() since
Python would do things like ask if a ZIP file is a terminal and get
confused when the old implementation mistakenly said yes, because the
fastest way to guarantee native file descriptors is to dup(2). This
change also improves the async signal safety of zipos and ensures it
doesn't maintain any open file descriptors beyond that which the user
has opened.

This change makes a lot of progress towards adding magic numbers that
are specific to platforms other than Linux. The philosophy here is that,
if you use an operating system like FreeBSD, then you should be able to
take advantage of FreeBSD exclusive features, even if we don't polyfill
them on other platforms. For example, you can now open() a file with the
O_VERIFY flag. If your program runs on other platforms, then Cosmo will
automatically set O_VERIFY to zero. This lets you safely use it without
the need for #ifdef or ifstatements which detract from readability.

One of the blindspots of the ASAN memory hardening we use to offer Rust
like assurances has always been that memory passed to the kernel via
system calls (e.g. writev) can't be checked automatically since the
kernel wasn't built with MODE=asan. This change makes more progress
ensuring that each system call will verify the soundness of memory
before it's passed to the kernel. The code for doing these checks is
fast, particularly for buffers, where it can verify 64 bytes a cycle.

- Correct O_LOOP definition on NT
- Introduce program_executable_name
- Add ASAN guards to more system calls
- Improve termios compatibility with BSDs
- Fix bug in Windows auxiliary value encoding
- Add BSD and XNU specific errnos and open flags
- Add check to ensure build doesn't talk to internet
2021-08-22 01:11:53 -07:00
Paul Kulchenko 2730c66f4a Add GetTime to redbean Lua (#255) 2021-08-22 00:59:47 -07:00
Paul Kulchenko c506584174 Don't hyphenate delta log lines (#253) 2021-08-19 09:43:01 -07:00
Justine Tunney ed485391da Reduce memory needed by ASAN morgue 2021-08-19 09:34:24 -07:00
Justine Tunney d5a37de435 Fix upstream Lua test breakage due to Linenoise 2021-08-19 09:34:24 -07:00
Justine Tunney 65f32fad52 Fix regression in _gclongjmp() 2021-08-19 09:34:24 -07:00
Justine Tunney da45c7c80b Clean up gdtoa code 2021-08-19 09:34:24 -07:00
Paul Kulchenko 7341336b1a Improve redbean docs (#249)
- Add Location to ProgramHeader exclusions
- Add ProgramHeader documentation
- Update ProgramRedirect to check location validity
2021-08-19 09:34:50 -07:00
Paul Kulchenko a63b147a93 Add file/line reporting to redbean LuaLog (#250) 2021-08-19 09:32:33 -07:00
Paul Kulchenko 0ba3617e7f Add ERRORF macro for kLogError log level (#252) 2021-08-19 04:14:08 -07:00
Justine Tunney 8af197560e Improve Libc by making Python work even better
Actually Portable Python is now outperforming the Python binaries
that come bundled with Linux distros, at things like HTTP serving.
You can now have a fully featured Python install in just one .com
file that runs on six operating systems and is about 10mb in size.
With tuning, the tiniest is ~1mb. We've got most of the libraries
working, including pysqlite, and the repl now feels very pleasant.
The things you can't do quite yet are: threads and shared objects
but that can happen in the future, if the community falls in love
with this project and wants to see it developed further. Changes:

- Add siginterrupt()
- Add sqlite3 to Python
- Add issymlink() helper
- Make GetZipCdir() faster
- Add tgamma() and finite()
- Add legacy function lutimes()
- Add readlink() and realpath()
- Use heap allocations when appropriate
- Reorganize Python into two-stage build
- Save Lua / Python shell history to dotfile
- Integrate Python Lib embedding into linkage
- Make isregularfile() and isdirectory() go faster
- Make Python shell auto-completion work perfectly
- Make crash reports work better if changed directory
- Fix Python+NT open() / access() flag overflow error
- Disable Python tests relating to \N{LONG NAME} syntax
- Have Python REPL copyright() show all notice embeddings

The biggest technical challenge at the moment is working around
when Python tries to be too clever about filenames.
2021-08-18 22:16:23 -07:00
Gautham 98ccbf44b1 Tell _frozen_importlib to consider bytecode first (#248) 2021-08-18 21:58:05 -07:00
Justine Tunney ebb8c85496 Experiment with making Python go faster
The goal is to put the compiled pyc files in the APE ZIP.
2021-08-18 21:57:11 -07:00
Paul Kulchenko 4486ad5c9e Add ServeRedirect function to redbean Lua (#246) 2021-08-17 14:26:33 -07:00
13629 changed files with 1212525 additions and 241906 deletions

No files matched your search

+28
View File
@@ -0,0 +1,28 @@
name: build
on:
push:
branches:
- "master"
- "flake"
pull_request:
branches:
- "master"
# run workflow manually from the Actions tab
workflow_dispatch:
jobs:
matrix_on_mode:
runs-on: ubuntu-latest
strategy:
matrix:
mode: ["", tiny, opt, rel]
steps:
- uses: actions/checkout@v3
- name: support ape bins
run: sudo sh -c "echo ':APE:M::MZqFpD::/bin/sh:' >/proc/sys/fs/binfmt_misc/register"
- name: make matrix
run: V=0 make -j2 MODE=${{ matrix.mode }}
+2 -1
View File
@@ -11,4 +11,5 @@ __pycache__
/TAGS
/bx_enh_dbg.ini
/tool/emacs/*.elc
/usr/share/dict/words
/perf.data
/perf.data.old
-6
View File
@@ -1,6 +0,0 @@
virt: lxd
os: linux
language: c
script: make -j4 V=0
notifications:
email: false
+4 -3
View File
@@ -15,7 +15,7 @@
"alignas(x)",
"alignof(x)",
"artificial=",
"nodiscard=",
"dontdiscard=",
"mayalias=",
"forceinline=",
"forcealign(x)=",
@@ -46,9 +46,10 @@
"testonly=",
"donothing=",
"nosideeffect=",
"unreachable=",
"unreachable=",,
"notpossible=",
"thatispacked=",
"nothrow=",
"dontthrow=",
"nocallback=",
"relegated=",
"hidden=",
+104 -69
View File
@@ -37,7 +37,7 @@
#
# # basic debugging
# make -j8 -O MODE=dbg o/dbg/examples/crashreport.com
# o/dbg/examples/crashreport.com
# o/examples/crashreport.com
# less examples/crashreport.c
#
# # extremely tiny binaries
@@ -59,10 +59,9 @@
#
# build/config.mk
SHELL = /bin/sh
HOSTS ?= freebsd openbsd netbsd rhel7 rhel5 xnu win7 win10
SANITY := $(shell build/sanitycheck $$PPID)
GNUMAKEFLAGS += --output-sync
SHELL = build/bootstrap/cocmd.com
HOSTS ?= freebsd openbsd netbsd rhel7 rhel5 win10 xnu
MAKEFLAGS += --no-builtin-rules
.SUFFIXES:
.DELETE_ON_ERROR:
@@ -70,7 +69,10 @@ GNUMAKEFLAGS += --output-sync
.PHONY: all o bins check test depend tags
all: o
o: o/$(MODE)/ape \
o: o/$(MODE)
o/$(MODE): \
o/$(MODE)/ape \
o/$(MODE)/dsp \
o/$(MODE)/net \
o/$(MODE)/libc \
@@ -79,6 +81,19 @@ o: o/$(MODE)/ape \
o/$(MODE)/examples \
o/$(MODE)/third_party
.STRICT = 1
.PLEDGE = stdio rpath wpath cpath fattr proc
.UNVEIL = \
libc/integral \
libc/disclaimer.inc \
rx:build/bootstrap \
rx:o/third_party/gcc \
/proc/self/status \
rw:/dev/null \
w:o/stack.log \
/etc/hosts \
~/.runit.psk
PKGS =
-include ~/.cosmo.mk
@@ -95,7 +110,6 @@ include libc/intrin/intrin.mk # │ You can use the stack
include libc/linux/linux.mk # │ You can manipulate arrays
include libc/tinymath/tinymath.mk # │ You can issue raw system calls
include third_party/compiler_rt/compiler_rt.mk # │
include libc/bits/bits.mk # │
include libc/str/str.mk # │
include third_party/xed/xed.mk # │
include third_party/zlib/zlib.mk # │
@@ -105,24 +119,24 @@ include libc/fmt/fmt.mk #─┘
include libc/calls/calls.mk #─┐
include libc/runtime/runtime.mk # ├──SYSTEMS RUNTIME
include libc/crt/crt.mk # │ You can issue system calls
include libc/rand/rand.mk # │
include libc/unicode/unicode.mk # │
include third_party/dlmalloc/dlmalloc.mk #─┘
include libc/mem/mem.mk #─┐
include libc/ohmyplus/ohmyplus.mk # ├──DYNAMIC RUNTIME
include libc/zipos/zipos.mk # │ You can now use stdio
include third_party/gdtoa/gdtoa.mk # │ You can finally call malloc()
include libc/time/time.mk # │
include libc/alg/alg.mk # │
include libc/zipos/zipos.mk # ├──DYNAMIC RUNTIME
include third_party/gdtoa/gdtoa.mk # │ You can now use stdio
include libc/time/time.mk # │ You can finally call malloc()
include libc/thread/thread.mk # │
include libc/stdio/stdio.mk # │
include third_party/libcxx/libcxx.mk # │
include net/net.mk # │
include libc/log/log.mk # │
include third_party/bzip2/bzip2.mk # │
include dsp/core/core.mk # │
include libc/x/x.mk # │
include third_party/stb/stb.mk # │
include dsp/scale/scale.mk # │
include dsp/mpeg/mpeg.mk # │
include dsp/dsp.mk # │
include third_party/zlib/gz/gz.mk # │
include third_party/musl/musl.mk # │
include third_party/getopt/getopt.mk # │
include libc/libc.mk #─┘
@@ -133,52 +147,72 @@ include net/http/http.mk # │
include third_party/mbedtls/mbedtls.mk # │
include net/https/https.mk # │
include third_party/regex/regex.mk #─┘
include third_party/tidy/tidy.mk
include third_party/third_party.mk
include libc/testlib/testlib.mk
include tool/viz/lib/vizlib.mk
include tool/args/args.mk
include test/tool/args/test.mk
include third_party/linenoise/linenoise.mk
include third_party/maxmind/maxmind.mk
include net/finger/finger.mk
include third_party/double-conversion/double-conversion.mk
include third_party/lua/lua.mk
include third_party/sed/sed.mk
include third_party/awk/awk.mk
include third_party/make/make.mk
include third_party/finger/finger.mk
include third_party/argon2/argon2.mk
include third_party/smallz4/smallz4.mk
include third_party/sqlite3/sqlite3.mk
include third_party/mbedtls/test/test.mk
include third_party/quickjs/quickjs.mk
include third_party/lz4cli/lz4cli.mk
include third_party/infozip/infozip.mk
include third_party/python/python.mk
include third_party/zip/zip.mk
include third_party/unzip/unzip.mk
include tool/build/lib/buildlib.mk
include third_party/chibicc/chibicc.mk
include third_party/chibicc/test/test.mk
include third_party/python/python.mk
include tool/build/emucrt/emucrt.mk
include tool/build/emubin/emubin.mk
include tool/build/build.mk
include examples/examples.mk
include examples/pyapp/pyapp.mk
include examples/pylife/pylife.mk
include tool/decode/lib/decodelib.mk
include tool/decode/decode.mk
include tool/lambda/lib/lib.mk
include tool/lambda/lambda.mk
include tool/plinko/lib/lib.mk
include tool/plinko/plinko.mk
include test/tool/plinko/test.mk
include tool/hash/hash.mk
include tool/net/net.mk
include tool/viz/viz.mk
include tool/tool.mk
include test/libc/alg/test.mk
include test/libc/tinymath/test.mk
include test/libc/intrin/test.mk
include test/libc/mem/test.mk
include test/libc/nexgen32e/test.mk
include test/libc/runtime/test.mk
include test/libc/thread/test.mk
include test/libc/sock/test.mk
include test/libc/bits/test.mk
include test/libc/str/test.mk
include test/libc/unicode/test.mk
include test/libc/log/test.mk
include test/libc/str/test.mk
include test/libc/calls/test.mk
include test/libc/x/test.mk
include test/libc/xed/test.mk
include test/libc/fmt/test.mk
include test/libc/dns/test.mk
include test/libc/rand/test.mk
include test/libc/time/test.mk
include test/libc/stdio/test.mk
include test/libc/release/test.mk
include test/libc/test.mk
include test/net/http/test.mk
include test/net/https/test.mk
include test/net/finger/test.mk
include test/net/test.mk
include test/tool/build/lib/test.mk
include test/tool/build/test.mk
@@ -205,55 +239,52 @@ CHECKS = $(foreach x,$(PKGS),$($(x)_CHECKS))
bins: $(BINS)
check: $(CHECKS)
test: $(TESTS) all
test: $(TESTS)
depend: o/$(MODE)/depend
tags: TAGS HTAGS
o/$(MODE)/.x:
@mkdir -p $(@D) && touch $@
@$(COMPILE) -AMKDIR -tT$@ $(MKDIR) $(@D)
ifneq ($(findstring 4.,,$(MAKE_VERSION)),$(MAKE_VERSION))
o/$(MODE)/srcs.txt: o/$(MODE)/.x $(MAKEFILES) $(call uniq,$(foreach x,$(SRCS),$(dir $(x))))
$(file >$@) $(foreach x,$(SRCS),$(file >>$@,$(x)))
$(file >$@,$(SRCS))
o/$(MODE)/hdrs.txt: o/$(MODE)/.x $(MAKEFILES) $(call uniq,$(foreach x,$(HDRS) $(INCS),$(dir $(x))))
$(file >$@) $(foreach x,$(HDRS) $(INCS),$(file >>$@,$(x)))
$(file >$@,$(HDRS) $(INCS))
o/$(MODE)/incs.txt: o/$(MODE)/.x $(MAKEFILES) $(call uniq,$(foreach x,$(INCS) $(INCS),$(dir $(x))))
$(file >$@) $(foreach x,$(INCS) $(INCS),$(file >>$@,$(x)))
else
o/$(MODE)/srcs.txt: o/$(MODE)/.x $(MAKEFILES) $(call uniq,$(foreach x,$(SRCS),$(dir $(x))))
$(MAKE) MODE=rel -j8 -pn bopit 2>/dev/null | sed -ne '/^SRCS/ {s/.*:= //;s/ */\n/g;p;q}' >$@
o/$(MODE)/hdrs.txt: o/$(MODE)/.x $(MAKEFILES) $(call uniq,$(foreach x,$(HDRS) $(INCS),$(dir $(x))))
$(MAKE) MODE=rel -j8 -pn bopit 2>/dev/null | sed -ne '/^HDRS/ {s/.*:= //;s/ */\n/g;p;q}' >$@
o/$(MODE)/incs.txt: o/$(MODE)/.x $(MAKEFILES) $(call uniq,$(foreach x,$(INCS) $(INCS),$(dir $(x))))
$(MAKE) MODE=rel -j8 -pn bopit 2>/dev/null | sed -ne '/^INCS/ {s/.*:= //;s/ */\n/g;p;q}' >$@
endif
$(file >$@,$(INCS))
o/$(MODE)/depend: o/$(MODE)/.x o/$(MODE)/srcs.txt o/$(MODE)/hdrs.txt o/$(MODE)/incs.txt $(SRCS) $(HDRS) $(INCS)
@$(COMPILE) -AMKDEPS $(MKDEPS) -o $@ -r o/$(MODE)/ o/$(MODE)/srcs.txt o/$(MODE)/hdrs.txt o/$(MODE)/incs.txt
@$(COMPILE) -AMKDEPS -L320 $(MKDEPS) -o $@ -r o/$(MODE)/ @o/$(MODE)/srcs.txt @o/$(MODE)/hdrs.txt @o/$(MODE)/incs.txt
TAGS: o/$(MODE)/srcs.txt $(SRCS)
@rm -f $@
@$(COMPILE) -ATAGS -T$@ $(TAGS) $(TAGSFLAGS) -L $< -o $@
o/$(MODE)/srcs-old.txt: o/$(MODE)/.x $(MAKEFILES) $(call uniq,$(foreach x,$(SRCS),$(dir $(x))))
$(file >$@) $(foreach x,$(SRCS),$(file >>$@,$(x)))
o/$(MODE)/hdrs-old.txt: o/$(MODE)/.x $(MAKEFILES) $(call uniq,$(foreach x,$(HDRS) $(INCS),$(dir $(x))))
$(file >$@) $(foreach x,$(HDRS) $(INCS),$(file >>$@,$(x)))
HTAGS: o/$(MODE)/hdrs.txt $(HDRS)
@rm -f $@
@$(COMPILE) -ATAGS -T$@ build/htags -L $< -o $@
TAGS: .UNSANDBOXED = 1
TAGS: o/$(MODE)/srcs-old.txt $(SRCS)
@$(RM) $@
@$(TAGS) $(TAGSFLAGS) -L $< -o $@
HTAGS: .UNSANDBOXED = 1
HTAGS: o/$(MODE)/hdrs-old.txt $(HDRS)
@$(RM) $@
@build/htags -L $< -o $@
loc: .UNSANDBOXED = 1
loc: o/$(MODE)/tool/build/summy.com
find -name \*.h -or -name \*.c -or -name \*.S | \
$(XARGS) wc -l | grep total | awk '{print $$1}' | $<
COSMOPOLITAN_OBJECTS = \
NET_HTTP \
LIBC_DNS \
LIBC_SOCK \
LIBC_NT_WS2_32 \
LIBC_NT_IPHLPAPI \
LIBC_NT_MSWSOCK \
LIBC_OHMYPLUS \
LIBC_X \
THIRD_PARTY_GETOPT \
LIBC_LOG \
LIBC_UNICODE \
LIBC_TIME \
LIBC_ZIPOS \
THIRD_PARTY_ZLIB \
@@ -261,27 +292,26 @@ COSMOPOLITAN_OBJECTS = \
LIBC_STDIO \
THIRD_PARTY_GDTOA \
THIRD_PARTY_REGEX \
LIBC_ALG \
LIBC_MEM \
THIRD_PARTY_DLMALLOC \
LIBC_BITS \
LIBC_RUNTIME \
LIBC_ELF \
LIBC_CALLS \
LIBC_RAND \
LIBC_SYSV_CALLS \
LIBC_NT_KERNELBASE \
LIBC_NT_SHELL32 \
LIBC_NT_PSAPI \
LIBC_NT_POWRPROF \
LIBC_NT_PDH \
LIBC_NT_GDI32 \
LIBC_NT_COMDLG32 \
LIBC_NT_NETAPI32 \
LIBC_NT_URL \
LIBC_NT_USER32 \
LIBC_NT_NTDLL \
LIBC_NT_ADVAPI32 \
LIBC_FMT \
THIRD_PARTY_COMPILER_RT \
LIBC_THREAD \
LIBC_TINYMATH \
THIRD_PARTY_XED \
LIBC_STR \
LIBC_SYSV \
LIBC_INTRIN \
@@ -289,9 +319,8 @@ COSMOPOLITAN_OBJECTS = \
LIBC_NEXGEN32E
COSMOPOLITAN_HEADERS = \
APE \
LIBC \
LIBC_ALG \
LIBC_BITS \
LIBC_CALLS \
LIBC_DNS \
LIBC_ELF \
@@ -301,40 +330,46 @@ COSMOPOLITAN_HEADERS = \
LIBC_MEM \
LIBC_NEXGEN32E \
LIBC_NT \
LIBC_OHMYPLUS \
LIBC_RAND \
LIBC_RUNTIME \
LIBC_SOCK \
LIBC_STDIO \
THIRD_PARTY_XED \
LIBC_STR \
LIBC_SYSV \
LIBC_THREAD \
LIBC_TIME \
LIBC_TINYMATH \
LIBC_UNICODE \
LIBC_X \
LIBC_ZIPOS \
NET_HTTP \
THIRD_PARTY_DLMALLOC \
THIRD_PARTY_GDTOA \
THIRD_PARTY_GETOPT \
THIRD_PARTY_MUSL \
THIRD_PARTY_ZLIB \
THIRD_PARTY_ZLIB_GZ \
THIRD_PARTY_REGEX
o/$(MODE)/cosmopolitan.a.txt:
printf "%s\n" $(call reverse,$(call uniq,$(foreach x,$(COSMOPOLITAN_OBJECTS),$($(x)))))
o/$(MODE)/cosmopolitan.a: $(filter-out o/libc/stubs/exit11.o,$(foreach x,$(COSMOPOLITAN_OBJECTS),$($(x)_A_OBJS)))
o/cosmopolitan.h: \
o/$(MODE)/tool/build/rollup.com \
libc/integral/normalize.inc \
$(foreach x,$(COSMOPOLITAN_HEADERS),$($(x)_HDRS))
@$(COMPILE) -AROLLUP -T$@ $^ >$@
o/$(MODE)/cosmopolitan.a: \
$(foreach x,$(COSMOPOLITAN_OBJECTS),$($(x)_A_OBJS))
o/cosmopolitan.h: \
o/$(MODE)/tool/build/rollup.com \
libc/integral/normalize.inc \
$(foreach x,$(COSMOPOLITAN_HEADERS),$($(x)_HDRS)) \
$(foreach x,$(COSMOPOLITAN_HEADERS),$($(x)_INCS))
$(file >$(TMPDIR)/$(subst /,_,$@),libc/integral/normalize.inc $(foreach x,$(COSMOPOLITAN_HEADERS),$($(x)_HDRS)))
@$(COMPILE) -AROLLUP -T$@ o/$(MODE)/tool/build/rollup.com @$(TMPDIR)/$(subst /,_,$@) >$@
o/cosmopolitan.html: \
o/$(MODE)/third_party/chibicc/chibicc.com.dbg \
$(filter-out %.s,$(foreach x,$(COSMOPOLITAN_OBJECTS),$($(x)_SRCS)))
$(filter-out %.s,$(foreach x,$(COSMOPOLITAN_OBJECTS),$($(x)_SRCS))) \
$(SRCS) \
$(HDRS)
$(file >$(TMPDIR)/$(subst /,_,$@),$(filter-out %.s,$(foreach x,$(COSMOPOLITAN_OBJECTS),$($(x)_SRCS))))
o/$(MODE)/third_party/chibicc/chibicc.com.dbg -J \
-fno-common -include libc/integral/normalize.inc -o $@ \
$(filter-out %.s,$(foreach x,$(COSMOPOLITAN_OBJECTS),$($(x)_SRCS)))
@$(TMPDIR)/$(subst /,_,$@)
$(SRCS): \
libc/integral/normalize.inc \
@@ -368,9 +403,9 @@ $(SRCS):
$(HDRS):
$(INCS):
.DEFAULT:
@echo >&2
@echo NOTE: deleting o/$(MODE)/depend because of an unspecified prerequisite: $@ >&2
@echo >&2
rm -f o/$(MODE)/depend
@$(ECHO) >&2
@$(ECHO) NOTE: deleting o/$(MODE)/depend because of an unspecified prerequisite: $@ >&2
@$(ECHO) >&2
$(RM) o/$(MODE)/depend
-include o/$(MODE)/depend
+193 -24
View File
@@ -1,5 +1,6 @@
![Cosmopolitan Honeybadger](usr/share/img/honeybadger.png)
[![build](https://github.com/jart/cosmopolitan/actions/workflows/build.yml/badge.svg)](https://github.com/jart/cosmopolitan/actions/workflows/build.yml)
# Cosmopolitan
[Cosmopolitan Libc](https://justine.lol/cosmopolitan/index.html) makes C
@@ -22,27 +23,90 @@ If you're doing your development work on Linux or BSD then you need just
five files to get started. Here's what you do on Linux:
```sh
wget https://justine.lol/cosmopolitan/cosmopolitan-amalgamation-1.0.zip
unzip cosmopolitan-amalgamation-1.0.zip
wget https://justine.lol/cosmopolitan/cosmopolitan-amalgamation-2.0.1.zip
unzip cosmopolitan-amalgamation-2.0.1.zip
printf 'main() { printf("hello world\\n"); }\n' >hello.c
gcc -g -Os -static -nostdlib -nostdinc -fno-pie -no-pie -mno-red-zone \
-fno-omit-frame-pointer -pg -mnop-mcount \
-o hello.com.dbg hello.c -fuse-ld=bfd -Wl,-T,ape.lds \
-include cosmopolitan.h crt.o ape.o cosmopolitan.a
-fno-omit-frame-pointer -pg -mnop-mcount -mno-tls-direct-seg-refs \
-o hello.com.dbg hello.c -fuse-ld=bfd -Wl,-T,ape.lds -Wl,--gc-sections \
-include cosmopolitan.h crt.o ape-no-modify-self.o cosmopolitan.a
objcopy -S -O binary hello.com.dbg hello.com
```
You now have a portable program. Please note that your APE binary will
assimilate itself as a conventional resident of your platform after the
first run, so it can be fast and efficient for subsequent executions.
You now have a portable program.
```sh
./hello.com
bash -c './hello.com' # zsh/fish workaround (we upstreamed patches)
bash -c './hello.com' # zsh/fish workaround (we patched them in 2021)
```
So if you intend to copy the binary to Windows or Mac then please do
that before you run it, not after.
If `./hello.com` executed on Linux throws an error about not finding an
interpreter, it should be fixed by running the following command (although
note that it may not survive a system restart):
```sh
sudo sh -c "echo ':APE:M::MZqFpD::/bin/sh:' >/proc/sys/fs/binfmt_misc/register"
```
If the same command produces puzzling errors on WSL or WINE when using
Redbean 2.x, they may be fixed by disabling binfmt_misc:
```sh
sudo sh -c 'echo -1 >/proc/sys/fs/binfmt_misc/status'
```
Since we used the `ape-no-modify-self.o` bootloader (rather than
`ape.o`) your executable will not modify itself when it's run. What
it'll instead do, is extract a 4kb program to `${TMPDIR:-${HOME:-.}}`
that maps your program into memory without needing to copy it. It's
possible to install the APE loader systemwide as follows.
```sh
# System-Wide APE Install
# for Linux, Darwin, and BSDs
# 1. Copies APE Loader to /usr/bin/ape
# 2. Registers w/ binfmt_misc too if Linux
ape/apeinstall.sh
# System-Wide APE Uninstall
# for Linux, Darwin, and BSDs
ape/apeuninstall.sh
```
It's also possible to convert APE binaries into the system-local format
by using the `--assimilate` flag. Please note that if binfmt_misc is in
play, you'll need to unregister it temporarily before doing this, since
the assimilate feature is part of the shell script header.
```sh
$ file hello.com
hello.com: DOS/MBR boot sector
./hello.com --assimilate
$ file hello.com
hello.com: ELF 64-bit LSB executable
```
Now that you're up and running with Cosmopolitan Libc and APE, here's
some of the most important troubleshooting tools APE offers that you
should know, in case you encounter any issues:
```sh
./hello.com --strace # log system calls to stderr
./hello.com --ftrace # log function calls to stderr
```
Do you love tiny binaries? If so, you may not be happy with Cosmo adding
heavyweight features like tracing to your binaries by default. In that
case, you may want to consider using our build system:
```sh
make -j8 MODE=tiny
```
Which will cause programs such as `hello.com` and `life.com` to shrink
from 60kb in size to about 16kb. There's also a prebuilt amalgamation
online <https://justine.lol/cosmopolitan/cosmopolitan-tiny.zip> hosted
on our download page <https://justine.lol/cosmopolitan/download.html>.
### MacOS
@@ -64,32 +128,137 @@ Windows](https://justine.lol/cosmopolitan/windows-compiling.html)
tutorial. It's needed because the ELF object format is what makes
universal binaries possible.
Cosmopolitan officially only builds on Linux. However, one highly
experimental (and currently broken) thing you could try, is building the
entire cosmo repository from source using the cross9 toolchain.
```
mkdir -p o/third_party
rm -rf o/third_party/gcc
wget https://justine.lol/linux-compiler-on-windows/cross9.zip
unzip cross9.zip
mv cross9 o/third_party/gcc
build/bootstrap/make.com
```
## Source Builds
Cosmopolitan can be compiled from source on any Linux distro. GNU make
needs to be installed beforehand. This is a freestanding hermetic
repository that bootstraps using a vendored static gcc9 executable.
No further dependencies are required.
Cosmopolitan can be compiled from source on any Linux distro. First, you
need to download or clone the repository.
```sh
wget https://justine.lol/cosmopolitan/cosmopolitan-1.0.tar.gz
tar xf cosmopolitan-1.0.tar.gz # see releases page
wget https://justine.lol/cosmopolitan/cosmopolitan.tar.gz
tar xf cosmopolitan.tar.gz # see releases page
cd cosmopolitan
make -j16
```
This will build the entire repository and run all the tests:
```sh
build/bootstrap/make.com -j16
o//examples/hello.com
find o -name \*.com | xargs ls -rShal | less
```
If you get an error running make.com then it's probably because you have
WINE installed to `binfmt_misc`. You can fix that by installing the the
APE loader as an interpreter. It'll improve build performance too!
```sh
ape/apeinstall.sh
```
Since the Cosmopolitan repository is very large, you might only want to
build a particular thing. Cosmopolitan's build config does a good job at
having minimal deterministic builds. For example, if you wanted to build
only hello.com then you could do that as follows:
```sh
build/bootstrap/make.com -j16 o//examples/hello.com
```
Sometimes it's desirable to build a subset of targets, without having to
list out each individual one. You can do that by asking make to build a
directory name. For example, if you wanted to build only the targets and
subtargets of the chibicc package including its tests, you would say:
```sh
build/bootstrap/make.com -j16 o//third_party/chibicc
o//third_party/chibicc/chibicc.com --help
```
Cosmopolitan provides a variety of build modes. For example, if you want
really tiny binaries (as small as 12kb in size) then you'd say:
```sh
build/bootstrap/make.com -j16 MODE=tiny
```
Here's some other build modes you can try:
```sh
build/bootstrap/make.com -j16 MODE=dbg # asan + ubsan + debug
build/bootstrap/make.com -j16 MODE=asan # production memory safety
build/bootstrap/make.com -j16 MODE=opt # -march=native optimizations
build/bootstrap/make.com -j16 MODE=rel # traditional release binaries
build/bootstrap/make.com -j16 MODE=optlinux # optimal linux-only performance
build/bootstrap/make.com -j16 MODE=tinylinux # tiniest linux-only 4kb binaries
```
For further details, see [//build/config.mk](build/config.mk).
## GDB
Here's the recommended `~/.gdbinit` config:
```gdb
set host-charset UTF-8
set target-charset UTF-8
set target-wide-charset UTF-8
set osabi none
set complaints 0
set confirm off
set history save on
set history filename ~/.gdb_history
define asm
layout asm
layout reg
end
define src
layout src
layout reg
end
src
```
You normally run the `.com.dbg` file under gdb. If you need to debug the
`.com` file itself, then you can load the debug symbols independently as
```
gdb foo.com -ex 'add-symbol-file foo.com.dbg 0x401000'
```
## Support Vector
| Platform | Min Version | Circa |
| :--- | ---: | ---: |
| AMD | K8 Venus | 2005 |
| Intel | Core | 2006 |
| New Technology | Vista | 2006 |
| GNU/Systemd | 2.6.18 | 2007 |
| XNU's Not UNIX! | 15.6 | 2018 |
| FreeBSD | 12 | 2018 |
| Windows | Vista | 2006 |
| Linux | 2.6.18 | 2007 |
| Mac OS X | 15.6 | 2018 |
| FreeBSD | 13 | 2020 |
| OpenBSD | 6.4 | 2018 |
| NetBSD | 9.1 | 2020 |
| GNU Make | 3.80 | 2010 |
| NetBSD | 9.2 | 2021 |
## Special Thanks
Funding for this project is crowdsourced using
[GitHub Sponsors](https://github.com/sponsors/jart) and
[Patreon](https://www.patreon.com/jart). Your support is what makes this
project possible. Thank you! We'd also like to give special thanks to
the following individuals:
- [Joe Drumgoole](https://github.com/jdrumgoole)
For publicly sponsoring our work at the highest tier.
+223 -164
View File
@@ -5,16 +5,16 @@
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ above copyright notice & this permission notice appear in all copies │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL │
│ THE AUTHOR BE LIABLE FOR ANY SPECIAL DIRECT INDIRECT, OR EVEN │
│ CONSEQUENTIAL DAMAGE OR ANY DAMAGES WHATSOEVER RESULTING FROM │
│ LOSS OF USE, DATA OR PROFITS WHETHER IN AN ACTION OF CONTRACT │
│ NEGLIGENCE OR OTHER TORTIOUS ACTION, THAT ARISE OUT OF, OR IN │
│ CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. │
╠──────────────────────────────────────────────────────────────────────────────╣
│░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░│
│░░░░░░░█▀█░█▀█░▀█▀░█░█░█▀█░█░░░█░░░█░█░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░│
@@ -51,7 +51,7 @@
.previous
.section .rodata,"a",@progbits
.align __SIZEOF_POINTER__
__ro: .endobj __ro,globl,hidden # ←for gdb readibility
__ro: .endobj __ro,globl,hidden # ←for gdb readability
.previous
.section .data,"aw",@progbits
.align __SIZEOF_POINTER__
@@ -60,7 +60,7 @@ __ro: .endobj __ro,globl,hidden # ←for gdb readibility
.align __SIZEOF_POINTER__
.previous
.section .rodata.str1.1,"a",@progbits
cstr: .endobj cstr,globl,hidden # ←for gdb readibility
cstr: .endobj cstr,globl,hidden # ←for gdb readability
.previous
.section .head,"ax",@progbits
@@ -193,12 +193,12 @@ stub: mov $0x40,%dl # *literally* dos
to a program as soon as its first sector is loaded. That gives
us control over user-facing latency, even though the next step
will generally be asking the BIOS to load more.
The process is trivial enough that this entrypoint can support
handoffs from alternative program-loaders e.g. Grub and MS-DOS
so long as they either load our full program, or implement the
PC BIOS disk service API.
Since so many different implementations of these APIs have been
built the last forty years these routines also canonicalize the
cpu and program state, as it is written in the System V ABI. */
@@ -250,6 +250,7 @@ pc: cld
xor %cx,%cx # current cylinder
xor %dh,%dh # current head
mov $v_ape_realsectors,%di # total sectors
sub $v_ape_realslacksectors,%di
3: call pcread
dec %di
jnz 3b
@@ -301,7 +302,21 @@ pc: cld
// @since IBM Personal Computer XT
dsknfo: push %bx
1: push %dx
mov $0x08,%ah # get disk params
mov $0x16,%ah # make sure there is disk in drive,
int $0x13 # by querying change-line status
jnc 2f
xor %ax,%ax # if error or change line active,
int $0x13 # do a reset...
jc 9f
mov $0x0201,%ax # ...then do a read, to confirm that
mov $0x0001,%cx # there is disk in drive
mov $0,%dh # (verify (%ah = 4) does not work
mov $IMAGE_BASE_REAL>>4,%bx # under QEMU, which always returns
mov %bx,%es # success)
xor %bx,%bx
int $0x13
jc 9f
2: mov $0x08,%ah # get disk params
int $0x13
jc 9f
mov %cl,%bh
@@ -540,50 +555,85 @@ apesh: .ascii "'\n#'\"\n" # sixth edition shebang
// modify the binary to follow the local system's convention.
// There isn't a one-size-fits-all approach for this, thus we
// present two choices.
.ascii "o=\"$(command -v \"$0\")\"\n"
// Try to use a system-wide APE loader.
.ascii "[ x\"$1\" != x--assimilate ] && "
.ascii "type ape >/dev/null 2>&1 && "
.ascii "exec ape \"$o\" \"$@\"\n"
#ifdef APE_LOADER
// There is no system-wide APE loader, but there is one
// embedded inside the APE. So if the system is not MacOs,
// extract the loader into a temp folder, and use it to
// load the APE without modifying it.
.ascii "[ x\"$1\" != x--assimilate ] && {\n"
.ascii "t=\"${TMPDIR:-${HOME:-.}}/.ape\"\n"
.ascii "[ -x \"$t\" ] || {\n"
.ascii "mkdir -p \"${t%/*}\" &&\n"
.ascii "dd if=\"$o\" of=\"$t.$$\" skip=\""
.shstub ape_loader_dd_skip,2
.ascii "\" count=\""
.shstub ape_loader_dd_count,2
.ascii "\" bs=64 2>/dev/null\n"
#if SupportsXnu()
.ascii "[ -d /Applications ] && "
.ascii "dd if=\"$t.$$\""
.ascii " of=\"$t.$$\""
.ascii " skip=6"
.ascii " count=10"
.ascii " bs=64"
.ascii " conv=notrunc"
.ascii " 2>/dev/null\n"
#endif /* SupportsXnu() */
.ascii "chmod 755 \"$t.$$\"\n"
.ascii "mv -f \"$t.$$\" \"$t\"\n"
.ascii "}\n"
.ascii "exec \"$t\" \"$o\" \"$@\"\n"
.ascii "}\n"
#endif /* APE_LOADER */
#ifndef APE_NO_MODIFY_SELF
// The default behavior is: to overwrite the header in place.
// We prefer this because it's a tiny constant one time cost.
// We simply printf a 64-byte header and call execve() again.
.ascii "o=\"$(command -v \"$0\")\"\n"
#ifdef APE_BUILDSAFE
// This code is intended for binaries in build/bootstrap/. It
// causes them to not self-modify since they're checked-in to
// version control. It assumes cosmopolitan's build prefix is
// being used by convention and that the program has the noop
// flag -n which is used to prime the binary.
.ascii "if [ ${o##o/} = $o ]; then\n"
.ascii " if ! [ o/$o -nt $o ]; then\n"
.ascii " mkdir -p o/${o%/*} || exit\n"
.ascii " cp -f $o o/$o.$$ || exit\n"
.ascii " o/$o.$$ -n || exit\n"
.ascii " mv -f o/$o.$$ o/$o || exit\n"
.ascii " fi\n"
.ascii " exec o/$o \"$@\"\n"
.ascii "fi\n"
#endif
#else
// The alternative behavior is to copy to $TMPDIR and edit.
// This imposes a variety of caveats of its own that should
// be considered by the user beforehand, such as whether or
// not /tmp is considered trustworthy on a given system, or
// if the administrator chose to mount it with noexec. It's
// up to the user to decide what's best in those situations
// and also note that argv[0] and getauxval(AT_EXECFN) will
// change as a result of this, and lastly note we don't try
// to cleanup the tmp copies for the sake of efficiency. It
// should also be noted that if $0 has directory components
// then permission clashes can happen between system users,
// since only root is able to set the sticky bit, which can
// be addressed simply by overriding the TMPDIR environment
.ascii "o=\"${TMPDIR:-/tmp}/$0\"\n"
.ascii "if [ ! -e \"$o\" ]; then\n"
.ascii "d=\"$o\"\n"
.ascii "o=\"$o.$$\"\n"
.ascii "mkdir -p \"${o%/*}\" 2>/dev/null\n"
.ascii "cp -f \"$0\" \"$o\" || exit 120\n"
// The alternative behavior is to copy to $TMPDIR or $HOME or
// the current directory. We like TMPDIR because it's part of
// the IEEE POSIX standard whereas alternatives (XDG) aren't.
.ascii "t=\"${TMPDIR:-${HOME:-.}}/$0\"\n"
.ascii "[ x\"$1\" != x--assimilate ] || [ ! -e \"$t\" ] && {\n"
.ascii "[ x\"$1\" != x--assimilate ] && {\n"
.ascii "mkdir -p \"${t%/*}\" 2>/dev/null\n"
.ascii "cp -f \"$o\" \"$t.$$\" &&\n"
.ascii "mv -f \"$t.$$\" \"$t\" || exit 120\n"
.ascii "o=\"$t\"\n"
.ascii "}\n"
#endif /* APE_NO_MODIFY_SELF */
.ascii "exec 7<> \"$o\" || exit 121\n"
.ascii "printf '"
.ascii "\\177ELF" # 0x0: ⌂ELF
.ascii "\\2" # 4: long mode
.ascii "\\1" # 5: little endian
.ascii "\\1" # 6: elf v1.o
.ascii "\\011" # 7: FreeBSD
.ascii "\\0" # 8: os/abi ver.
.ascii "\\0\\0\\0" # 9: padding 3/7
.ascii "\\0\\0\\0\\0" # padding 4/7
.ascii "\\2\\0" # 10: εxεcµταblε
.ascii "\\076\\0" # 12: NexGen32e
.ascii "\\1\\0\\0\\0" # 14: elf v1.o
.shstub ape_elf_entry,8 # 18: e_entry
.shstub ape_elf_phoff,8 # 20: e_phoff
.shstub ape_elf_shoff,8 # 28: e_shoff
.ascii "\\0\\0\\0\\0" # 30: e_flags
.ascii "\\100\\0" # 34: e_ehsize
.ascii "\\070\\0" # 36: e_phentsize
.shstub ape_elf_phnum,2 # 38: e_phnum
.ascii "\\0\\0" # 3a: e_shentsize
.shstub ape_elf_shnum,2 # 3c: e_shnum
.shstub ape_elf_shstrndx,2 # 3e: e_shstrndx
.ascii "' >&7\n"
.ascii "exec 7<&-\n"
#if SupportsXnu()
.ascii "if [ -d /Applications ]; then\n"
.ascii "[ -d /Applications ] && "
.ascii "dd if=\"$o\""
.ascii " of=\"$o\""
.ascii " bs=8"
@@ -592,63 +642,43 @@ apesh: .ascii "'\n#'\"\n" # sixth edition shebang
.ascii "\" count=\""
.shstub ape_macho_dd_count,2
.ascii "\" conv=notrunc 2>/dev/null\n"
.ascii "el"
#endif /* XNU */
.ascii "if exec 7<> \"$o\"; then\n"
.ascii "printf '"
.ascii "\\177ELF" # 0x0: ⌂ELF
.ascii "\\2" # 4: long mode
.ascii "\\1" # 5: little endian
.ascii "\\1" # 6: elf v1.o
.ascii "\\011" # 7: FreeBSD
.ascii "\\0" # 8: os/abi ver.
.ascii "\\0\\0\\0" # 9: padding 3/7
.ascii "\\0\\0\\0\\0" # padding 4/7
.ascii "\\2\\0" # 10: εxεcµταblε
.ascii "\\076\\0" # 12: NexGen32e
.ascii "\\1\\0\\0\\0" # 14: elf v1.o
.shstub ape_elf_entry,8 # 18: e_entry
.shstub ape_elf_phoff,8 # 20: e_phoff
.shstub ape_elf_shoff,8 # 28: e_shoff
.ascii "\\0\\0\\0\\0" # 30: e_flags
.ascii "\\100\\0" # 34: e_ehsize
.ascii "\\070\\0" # 36: e_phentsize
.shstub ape_elf_phnum,2 # 38: e_phnum
.ascii "\\0\\0" # 3a: e_shentsize
.shstub ape_elf_shnum,2 # 3c: e_shnum
.shstub ape_elf_shstrndx,2 # 3e: e_shstrndx
.ascii "' >&7\n"
.ascii "exec 7<&-\n"
.ascii "else\n"
.ascii "exit 121\n"
.ascii "fi\n"
.ascii "[ x\"$1\" = x--assimilate ] && exit 0\n"
#ifndef APE_NO_MODIFY_SELF
.ascii "exec \"$0\" \"$@\"\n" # optimistic execution
.ascii "exec \"$0\" \"$@\"\n" # try to preserve argv[0]
#else
.ascii "mv -f \"$o\" \"$d\" 2>/dev/null\n"
.ascii "o=\"$d\"\n"
.ascii "fi\n"
.ascii "}\n"
.ascii "o=\"$t\"\n"
.ascii "exec \"$o\" \"$@\"\n"
#endif /* APE_NO_MODIFY_SELF */
.ascii "R=$?\n"
.ascii "\n"
.ascii "if [ $R -eq 126 ] && [ \"$(uname -m)\" != x86_64 ]; then\n"
.ascii "if Q=\"$(command -v qemu-x86_64)\"; then\n"
.ascii "if [ \"$(uname -m)\" != x86_64 ]; then\n"
.ascii "Q=\"$(command -v qemu-x86_64)\" &&\n"
.ascii "exec \"$Q\" \"$o\" \"$@\"\n"
.ascii "else\n"
.ascii "echo error: need qemu-x86_64 >&2\n"
.ascii "fi\n"
#ifndef APE_NO_MODIFY_SELF
.ascii "elif [ $R -eq 127 ]; then\n" # means argv[0] was wrong
.ascii "else\n" # means argv[0] was wrong
.ascii " exec \"$o\" \"$@\"\n" # so do a path resolution
#endif /* APE_NO_MODIFY_SELF */
.ascii "fi\n"
.ascii "exit $R\n"
.endobj apesh
#ifdef APE_LOADER
.section .ape.loader,"a",@progbits
.align 64
ape_loader:
.incbin APE_LOADER
.endobj ape_loader,globl
.align 64
ape_loader_end:
nop
.endobj ape_loader_end,globl
.previous
#endif /* APE_LOADER */
#endif /* SupportsWindows() || SupportsMetal() || SupportsXnu() */
#if SupportsSystemv() || SupportsMetal()
.section .elf.phdrs,"a",@progbits
.long PT_LOAD
.long PF_R|PF_X
.stub ape_rom_offset,quad
@@ -657,6 +687,7 @@ apesh: .ascii "'\n#'\"\n" # sixth edition shebang
.stub ape_rom_filesz,quad
.stub ape_rom_memsz,quad
.stub ape_rom_align,quad
.long PT_LOAD
.long PF_R|PF_W
.stub ape_ram_offset,quad
@@ -665,14 +696,32 @@ apesh: .ascii "'\n#'\"\n" # sixth edition shebang
.stub ape_ram_filesz,quad
.stub ape_ram_memsz,quad
.stub ape_ram_align,quad
// APE Stack Configuration
//
// We actually respect this when allocating a deterministically
// addressed stack on Windows and other operating systems. However
// there's a few caveats:
//
// - If ape_stack_pf has PF_X then OpenBSD probably won't work
// - We don't bother creating a deterministic stack in MODE=tiny
//
// With that said, here's some examples of configuration:
//
// STATIC_SYMBOL("ape_stack_pf", "7"); // RWX
// STATIC_STACK_ADDR(0x6fffffff0000);
// STATIC_STACK_SIZE(65536);
//
// @see ape.lds for defaults
.long PT_GNU_STACK
.long PF_R|PF_W
.stub ape_stack_offset,quad
.stub ape_stack_vaddr,quad
.stub ape_stack_paddr,quad
.stub ape_stack_filesz,quad
.stub ape_stack_memsz,quad
.stub ape_stack_align,quad
.stub ape_stack_pf,long # override w/ PF_X for execstack
.stub ape_stack_offset,quad # ignored
.stub ape_stack_vaddr,quad # is mmap()'d with MAP_FIXED
.stub ape_stack_paddr,quad # ignored
.stub ape_stack_filesz,quad # ignored
.stub ape_stack_memsz,quad # is mmap(size) argument
.stub ape_stack_align,quad # must be 16+
#if SupportsOpenbsd() || SupportsNetbsd()
.long PT_NOTE
.long PF_R
@@ -825,7 +874,7 @@ ape_macho:
.long (520f-510f)/4 # count
510: .quad 0 # rax
.quad IMAGE_BASE_VIRTUAL # rbx
.quad 0 # rcx
.quad XNU # rcx
.quad 0 # rdx
.quad 0 # rdi
.quad 0 # rsi
@@ -839,7 +888,7 @@ ape_macho:
.quad 0 # r13
.quad 0 # r14
.quad 0 # r15
.quad _xnu # rip
.quad _start # rip
.quad 0 # rflags
.quad 0 # cs
.quad 0 # fs
@@ -876,7 +925,7 @@ ape_macho:
│ αcτµαlly pδrταblε εxεcµταblε § the new technology ─╬─│┼
╚────────────────────────────────────────────────────────────────────────────│─╝
The Portable Executable Format
@see https://docs.microsoft.com/en-us/windows/desktop/debug/pe-format
@see "The Portable Executable File Format from Top to Bottom",
Randy Kath, Microsoft Developer Network Technology Group. */
@@ -923,7 +972,7 @@ DLLEXE = DLLSTD
// ││││││││ │ │┌6:Contains Initialized Data
// ││││││││ o │ ││┌5:Contains Code
// ││││││││┌┴─┐rrrr│ ooror│││rorrr
PETEXT = 0b01110000000000000000000001100000
PETEXT = 0b01100000000000000000000001100000
PEDATA = 0b11000000000000000000000011000000
PEIMPS = 0b11000000000000000000000001000000
@@ -1059,18 +1108,12 @@ str.error:
str.crlf:
.asciz "\r\n"
.endobj str.crlf
str.cpuid:
.asciz "cpuid"
.endobj str.cpuid
str.oldskool:
.asciz "oldskool"
.endobj str.oldskool
str.e820:
.asciz "e820"
.endobj str.e820
str.long:
.asciz "nolong"
.endobj str.long
str.oldcpu:
.asciz "oldcpu"
.endobj str.oldcpu
// Serial Line Configuration (8250 UART 16550)
// If it's hacked, it'll at least get hacked very slowly.
@@ -1131,47 +1174,6 @@ gdt: .short 2f-1f # table byte length
.quad 0b0000000010101111100100110000000000000000000000001111111111111111 #48
2: .endobj gdt,global,hidden
/*─────────────────────────────────────────────────────────────────────────────╗
│ αcτµαlly pδrταblε εxεcµταblε § multiboot stub │
╚──────────────────────────────────────────────────────────────────────────────╝
boot modernized for the nineties */
#define GRUB_MAGIC 0x1BADB002
#define GRUB_EAX 0x2BADB002
#define GRUB_AOUT (1 << 16)
#define GRUB_CHECKSUM(FLAGS) (-(GRUB_MAGIC + (FLAGS)) & 0xffffffff)
// Grub Header.
.align 4
ape_grub:
.long GRUB_MAGIC # Magic
.long GRUB_AOUT # Flags
.long GRUB_CHECKSUM(GRUB_AOUT) # Checksum
.long RVA(ape_grub) # HeaderPhysicalAddress
.long IMAGE_BASE_PHYSICAL # TextPhysicalAddress
.long PHYSICAL(_edata) # LoadEndPhysicalAddress
.long PHYSICAL(_end) # BssEndPhysicalAddress
.long RVA(ape_grub_entry) # EntryPhysicalAddress
.endobj ape_grub,globl
// Grub Entrypoint.
// Takes CPU out of legacy mode and jumps to normal entrypoint.
// @noreturn
.align 4
ape_grub_entry:
.code32
// cmp $GRUB_EAX,%eax
// jne triplf
push $0
popf
mov $0x40,%dl
mov %cr0,%eax
and $~CR0_PE,%eax
mov %eax,%cr0
ljmpw $0,$REAL(pc)
.code16
.endfn ape_grub_entry
/*─────────────────────────────────────────────────────────────────────────────╗
│ αcτµαlly pδrταblε εxεcµταblε § real mode │
╚──────────────────────────────────────────────────────────────────────────────╝
@@ -1253,7 +1255,7 @@ longmodeloader:
lcheck: pushf # check for i8086 / i8088 / i80186
pop %ax
test $0x80,%ah # see intel manual volume 1 20.1.2
jnz 9f # we now assume 32bit is supported
jnz 10f # we now assume 32bit is supported
pushfl # now check for i386 or early i486
pop %eax # test ability to change cpuid bit
mov %eax,%ecx
@@ -1264,7 +1266,7 @@ lcheck: pushf # check for i8086 / i8088 / i80186
pushfl
pop %eax
cmp %eax,%ecx
je 12f # we assume cpuid inst is available
je 10f # we assume cpuid inst is available
or %ebx,%eax # puts cpuid bit in the on position
push %eax
popfl
@@ -1282,11 +1284,7 @@ lcheck: pushf # check for i8086 / i8088 / i80186
jne 10f
xor %ax,%ax
1: ret
9: mov $REAL(str.oldskool),%di
jmp 20f
10: mov $REAL(str.long),%di
jmp 20f
12: mov $REAL(str.cpuid),%di
10: mov $REAL(str.oldcpu),%di
20: call rldie
.endfn lcheck
@@ -1452,6 +1450,47 @@ long: push $GDT_LONG_DATA
jmp *%rax
.endfn long
/*─────────────────────────────────────────────────────────────────────────────╗
│ αcτµαlly pδrταblε εxεcµταblε § multiboot stub │
╚──────────────────────────────────────────────────────────────────────────────╝
boot modernized for the nineties */
#define GRUB_MAGIC 0x1BADB002
#define GRUB_EAX 0x2BADB002
#define GRUB_AOUT (1 << 16)
#define GRUB_CHECKSUM(FLAGS) (-(GRUB_MAGIC + (FLAGS)) & 0xffffffff)
// Grub Header.
.align 4
ape_grub:
.long GRUB_MAGIC # Magic
.long GRUB_AOUT # Flags
.long GRUB_CHECKSUM(GRUB_AOUT) # Checksum
.long RVA(ape_grub) # HeaderPhysicalAddress
.long IMAGE_BASE_PHYSICAL # TextPhysicalAddress
.long PHYSICAL(_edata) # LoadEndPhysicalAddress
.long PHYSICAL(_end) # BssEndPhysicalAddress
.long RVA(ape_grub_entry) # EntryPhysicalAddress
.endobj ape_grub,globl
// Grub Entrypoint.
// Takes CPU out of legacy mode and jumps to normal entrypoint.
// @noreturn
.align 4
ape_grub_entry:
.code32
// cmp $GRUB_EAX,%eax
// jne triplf
push $0
popf
mov $0x40,%dl
mov %cr0,%eax
and $~CR0_PE,%eax
mov %eax,%cr0
ljmpw $0,$REAL(pc)
.code64
.endfn ape_grub_entry
/* ▄▄▒▀▀▀▀▒▒▄
█████▓▓▄░░░░ ▒▒▄
▐█▓▓█▓▄█████▓░ ▀▒▄
@@ -1487,12 +1526,9 @@ kernel: movabs $ape_stack_vaddr,%rsp
.byte 0x0f,0x1f,0207 # nop rdi binbase
.long (IMAGE_BASE_VIRTUAL-IMAGE_BASE_REAL)/512
#endif
.weak __hostos
ezlea __hostos,ax
test %rax,%rax
jz 1f
movb $METAL,(%rax)
1: push $0
push $METAL # sets __hostos in crt.S
pop %rcx
push $0
mov %rsp,%rbp
mov .Lenv0(%rip),%rax
mov %rax,(%rbp) # envp[0][0]
@@ -1508,7 +1544,6 @@ kernel: movabs $ape_stack_vaddr,%rsp
push $1 # argc
xor %ebp,%ebp
xor %eax,%eax
xor %ecx,%ecx
xor %edx,%edx
xor %edi,%edi
xor %esi,%esi
@@ -1533,6 +1568,7 @@ kernel: movabs $ape_stack_vaddr,%rsp
.ldsvar _end
.ldsvar _etext
.ldsvar v_ape_realsectors
.ldsvar v_ape_realslacksectors
.ldsvar v_ape_highsectors
.ldsvar ape_idata_ro
.ldsvar ape_pad_rodata
@@ -1549,6 +1585,7 @@ kernel: movabs $ape_stack_vaddr,%rsp
.type __test_end,@object
.section .commentprologue,"a",@progbits
.globl kLegalNotices
.type kLegalNotices,@object
.hidden kLegalNotices
kLegalNotices:/*
@@ -1576,6 +1613,7 @@ ape_pad_text:
.type ape_pad_privileged,@object
.hidden ape_pad_privileged
ape_pad_privileged:
.align 4096
.previous
.section .ape.pad.rodata,"a",@progbits
@@ -1603,5 +1641,26 @@ ape_idata_ro:
__data_start:
.previous
.section .dataepilogue,"aw",@progbits
.type __data_end,@object
.globl __data_end
.hidden __data_end
__data_end:
.previous
.section .bssprologue,"aw",@nobits
.type __bss_start,@object
.globl __bss_start
.hidden __bss_start
__bss_start:
.previous
.section .bssepilogue,"aw",@nobits
.type __bss_end,@object
.globl __bss_end
.hidden __bss_end
__bss_end:
.previous
.end

+105 -45
View File
@@ -177,16 +177,20 @@
#include "ape/macros.internal.h"
#include "ape/relocations.h"
#include "libc/dce.h"
#include "libc/elf/def.h"
#include "libc/elf/pf2prot.internal.h"
#include "libc/nt/pedef.internal.h"
#include "libc/zip.h"
ENTRY(_start)
PHDRS {
Head PT_LOAD FLAGS(5);
Rom PT_LOAD FLAGS(5);
Ram PT_LOAD FLAGS(6);
stack PT_GNU_STACK FLAGS(6);
Head PT_LOAD FLAGS(PF_X|PF_R);
Rom PT_LOAD FLAGS(PF_X|PF_R);
Ram PT_LOAD FLAGS(PF_W|PF_R);
Tls PT_TLS FLAGS(PF_W|PF_R);
Bss PT_LOAD FLAGS(PF_W|PF_R);
stack PT_GNU_STACK FLAGS(PF_W|PF_R);
}
SECTIONS {
@@ -237,13 +241,16 @@ SECTIONS {
/*BEGIN: nt addressability guarantee */
.text . : {
BYTE(0x90) /* TODO: fix blinkenlights symbol __map_phdrs */
/* Code that needs to be addressable in Real Mode */
*(.text.real)
KEEP(*(SORT_BY_NAME(.sort.text.real.*)))
/* Code we want earlier in the binary w/o modifications */
KEEP(*(.ape.loader))
HIDDEN(_ereal = .);
. += 1;
/*END: realmode addressability guarantee */
/*BEGIN: morphable code */
. += 1;
/* Normal Code */
*(.start)
@@ -280,6 +287,7 @@ SECTIONS {
/* Privileged code invulnerable to magic */
KEEP(*(.ape.pad.privileged));
. += . > 0 ? 1 : 0;
/*END: morphable code */
HIDDEN(__privileged_start = .);
. += . > 0 ? 1 : 0;
*(.privileged)
@@ -289,6 +297,9 @@ SECTIONS {
/*BEGIN: Read Only Data */
KEEP(*(.ape.pad.rodata));
KEEP(*(.rodata.pytab.0));
KEEP(*(.rodata.pytab.1));
KEEP(*(.rodata.pytab.2));
*(.rodata .rodata.*)
*(.ubsan.types)
@@ -339,8 +350,10 @@ SECTIONS {
/*END: Read Only Data */
} :Rom
.data . : {
.data ALIGN(PAGESIZE) : {
/*BEGIN: Read/Write Data */
KEEP(*(SORT_BY_NAME(.piro.data.sort.iat.*)))
/*BEGIN: NT FORK COPYING */
KEEP(*(.dataprologue))
*(.data .data.*)
KEEP(*(SORT_BY_NAME(.sort.data.*)))
@@ -361,23 +374,44 @@ SECTIONS {
. = ALIGN(__SIZEOF_POINTER__);
KEEP(*(SORT_BY_NAME(.piro.data.sort.*)))
KEEP(*(.piro.pad.data))
. = ALIGN(PAGESIZE);
KEEP(*(.dataepilogue))
/*END: NT FORK COPYING */
HIDDEN(_edata = .);
PROVIDE_HIDDEN(edata = .);
KEEP(*(SORT_BY_NAME(.zip.*)))
HIDDEN(_ezip = .);
. = ALIGN(PAGESIZE);
} :Ram
. = ALIGN(PAGESIZE);
.tdata . : {
_tdata_start = .;
*(SORT_BY_ALIGNMENT(.tdata))
*(SORT_BY_ALIGNMENT(.tdata.*))
. = ALIGN(16);
_tdata_end = .;
. = ALIGN(PAGESIZE);
} :Tls :Ram
. = ALIGN(PAGESIZE);
/*END: file content that's loaded by o/s */
/*BEGIN: bss memory void */
.zip . : {
KEEP(*(SORT_BY_NAME(.zip.*)))
HIDDEN(_ezip = .);
}
.tbss . : {
_tbss_start = .;
*(SORT_BY_ALIGNMENT(.tbss))
*(SORT_BY_ALIGNMENT(.tbss.*))
. = ALIGN(16);
/* the %fs register is based on this location */
_tbss_end = .;
} :Tls
/*END: file content */
/*BEGIN: bss memory that's addressable */
.bss ALIGN(64) : {
/*BEGIN: NT FORK COPYING */
KEEP(*(.bssprologue))
KEEP(*(SORT_BY_NAME(.piro.bss.init.*)))
*(.piro.bss)
KEEP(*(SORT_BY_NAME(.piro.bss.sort.*)))
@@ -392,10 +426,12 @@ SECTIONS {
KEEP(*(SORT_BY_NAME(.sort.bss.*)))
KEEP(*(.bssepilogue))
/*END: NT FORK COPYING */
. = ALIGN(FRAMESIZE); /* for brk()/sbrk() allocation */
HIDDEN(_end = .);
PROVIDE_HIDDEN(end = .);
}
} :Bss
/*END: nt addressability guarantee */
/*END: bsd addressability guarantee */
@@ -451,9 +487,13 @@ PFSTUB4(ape_elf_phnum, (ape_phdrs_end - ape_phdrs) / 56);
PFSTUB4(ape_elf_shnum, 0);
PFSTUB4(ape_elf_shstrndx, 0);
HIDDEN(__privileged_addr = ROUNDDOWN(__privileged_start, PAGESIZE));
HIDDEN(_tdata_size = _tdata_end - _tdata_start);
HIDDEN(_tls_size = _tbss_end - _tdata_start);
HIDDEN(_tls_content = (_tdata_end - _tdata_start) + (_tbss_end - _tbss_start));
HIDDEN(__privileged_addr = ROUNDDOWN(__privileged_start, PAGESIZE));
HIDDEN(__privileged_size = (ROUNDUP(__privileged_end, PAGESIZE) -
ROUNDDOWN(__privileged_start, PAGESIZE)));
ROUNDDOWN(__privileged_start, PAGESIZE)));
HIDDEN(ape_rom_offset = 0);
HIDDEN(ape_rom_vaddr = ADDR(.head));
@@ -466,16 +506,18 @@ HIDDEN(ape_rom_rva = RVA(ape_rom_vaddr));
HIDDEN(ape_ram_offset = ape_rom_offset + ape_rom_filesz);
HIDDEN(ape_ram_vaddr = ADDR(.data));
HIDDEN(ape_ram_paddr = LOADADDR(.data));
HIDDEN(ape_ram_filesz = SIZEOF(.data));
HIDDEN(ape_ram_filesz = SIZEOF(.data) + SIZEOF(.tdata));
HIDDEN(ape_ram_memsz = ADDR(.bss) + SIZEOF(.bss) - ape_ram_vaddr);
HIDDEN(ape_ram_align = PAGESIZE);
HIDDEN(ape_ram_rva = RVA(ape_ram_vaddr));
HIDDEN(ape_stack_pf = DEFINED(ape_stack_pf) ? ape_stack_pf : PF_R | PF_W);
HIDDEN(ape_stack_prot = _PF2PROT(ape_stack_pf));
HIDDEN(ape_stack_offset = ape_ram_offset + ape_ram_filesz);
HIDDEN(ape_stack_vaddr = 0x700000000000 - STACKSIZE);
HIDDEN(ape_stack_vaddr = DEFINED(ape_stack_vaddr) ? ape_stack_vaddr : 0x700000000000);
HIDDEN(ape_stack_paddr = ape_ram_paddr + ape_ram_filesz);
HIDDEN(ape_stack_filesz = 0);
HIDDEN(ape_stack_memsz = STACKSIZE);
HIDDEN(ape_stack_memsz = DEFINED(ape_stack_memsz) ? ape_stack_memsz : STACKSIZE);
HIDDEN(ape_stack_align = 16);
HIDDEN(ape_note_offset = ape_rom_offset + (ape_note - ape_rom_vaddr));
@@ -496,8 +538,8 @@ HIDDEN(ape_text_rva = RVA(ape_text_vaddr));
HIDDEN(ape_data_offset = ape_ram_offset + LOADADDR(.data) - ape_ram_paddr);
HIDDEN(ape_data_paddr = LOADADDR(.data));
HIDDEN(ape_data_vaddr = ADDR(.data));
HIDDEN(ape_data_filesz = SIZEOF(.data));
HIDDEN(ape_data_memsz = SIZEOF(.data));
HIDDEN(ape_data_filesz = SIZEOF(.data) + SIZEOF(.tdata));
HIDDEN(ape_data_memsz = SIZEOF(.data) + SIZEOF(.tdata));
HIDDEN(ape_data_align = PAGESIZE);
HIDDEN(ape_data_rva = RVA(ape_data_vaddr));
@@ -508,36 +550,22 @@ HIDDEN(ape_bss_filesz = 0);
HIDDEN(ape_bss_memsz = SIZEOF(.bss));
HIDDEN(ape_bss_align = PAGESIZE);
#if SupportsXnu()
SHSTUB2(ape_macho_dd_skip, RVA(ape_macho) / 8);
SHSTUB2(ape_macho_dd_count, (ape_macho_end - ape_macho) / 8);
#endif
#if SupportsWindows() || SupportsMetal()
#define LINK_WINDOWS (SupportsWindows() && !DEFINED(EfiMain))
PFSTUB4(ape_pe_offset, ape_pe - ape_mz);
HIDDEN(ape_pe_optsz = ape_pe_sections - (ape_pe + 24));
HIDDEN(ape_pe_shnum = (ape_pe_sections_end - ape_pe_sections) / 40);
HIDDEN(ape_pe_base = IMAGE_BASE_VIRTUAL);
HIDDEN(ape_idata = LINK_WINDOWS ? RVA(ape_idata_iat) : 0);
HIDDEN(ape_idata_iatsize = LINK_WINDOWS ? ape_idata_iatend - ape_idata_iat : 0);
HIDDEN(ape_idata = LINK_WINDOWS ? RVA(ape_idata_idt) : 0);
HIDDEN(ape_idata_idtsize = LINK_WINDOWS ? ape_idata_idtend - ape_idata_idt : 0);
HIDDEN(v_ntversion = LINK_WINDOWS ? 6 : 1);
HIDDEN(v_ntdllchar = LINK_WINDOWS ? 288 : 0);
HIDDEN(v_ntsubversion = LINK_WINDOWS ? 6 : 5);
HIDDEN(v_ntsubsystem = (LINK_WINDOWS
? (DEFINED(GetMessage)
? kNtImageSubsystemWindowsGui
: kNtImageSubsystemWindowsCui)
: kNtImageSubsystemEfiApplication));
HIDDEN(ape_pe_entry = LINK_WINDOWS ? WinMain : EfiMain);
#endif
/* we roundup here because xnu wants the file load segments page-aligned */
/* but we don't want to add the nop padding to the ape program, so we'll */
/* let ape.S dd read past the end of the file into the wrapping binaries */
SHSTUB2(ape_loader_dd_skip, DEFINED(ape_loader) ? RVA(ape_loader) / 64 : 0);
SHSTUB2(ape_loader_dd_count,
DEFINED(ape_loader_end)
? ROUNDUP(ape_loader_end - ape_loader, PAGESIZE) / 64
: 0);
#if SupportsMetal()
HIDDEN(v_ape_realsectors =
MIN(0x70000 - IMAGE_BASE_REAL,
ROUNDUP(RVA(_edata), 4096)) / 512);
HIDDEN(v_ape_realslacksectors =
v_ape_realsectors - MIN(0x70000 - IMAGE_BASE_REAL,
ROUNDUP(RVA(_edata), 512)) / 512);
HIDDEN(v_ape_realpages = v_ape_realsectors / (4096 / 512));
HIDDEN(v_ape_highsectors =
(ROUNDUP(RVA(_edata), 512) / 512) - v_ape_realsectors);
@@ -640,6 +668,32 @@ CHURN(WinMain);
#endif /* SupportsWindows() */
#endif /* SupportsXnu() */
#if SupportsWindows() || SupportsMetal()
#define LINK_WINDOWS (SupportsWindows() && !DEFINED(EfiMain))
PFSTUB4(ape_pe_offset, ape_pe - ape_mz);
HIDDEN(ape_pe_optsz = ape_pe_sections - (ape_pe + 24));
HIDDEN(ape_pe_shnum = (ape_pe_sections_end - ape_pe_sections) / 40);
HIDDEN(ape_pe_base = IMAGE_BASE_VIRTUAL);
HIDDEN(ape_idata = LINK_WINDOWS ? RVA(ape_idata_iat) : 0);
HIDDEN(ape_idata_iatsize = LINK_WINDOWS ? ape_idata_iatend - ape_idata_iat : 0);
HIDDEN(ape_idata = LINK_WINDOWS ? RVA(ape_idata_idt) : 0);
HIDDEN(ape_idata_idtsize = LINK_WINDOWS ? ape_idata_idtend - ape_idata_idt : 0);
HIDDEN(v_ntversion = LINK_WINDOWS ? 6 : 1);
HIDDEN(v_ntdllchar = LINK_WINDOWS ? 288 : 0);
HIDDEN(v_ntsubversion = LINK_WINDOWS ? 6 : 5);
HIDDEN(v_ntsubsystem = (LINK_WINDOWS
? (DEFINED(GetMessage)
? kNtImageSubsystemWindowsGui
: kNtImageSubsystemWindowsCui)
: kNtImageSubsystemEfiApplication));
HIDDEN(ape_pe_entry = LINK_WINDOWS ? WinMain : EfiMain);
#endif
#if SupportsXnu()
SHSTUB2(ape_macho_dd_skip, RVA(ape_macho) / 8);
SHSTUB2(ape_macho_dd_count, (ape_macho_end - ape_macho) / 8);
#endif
ASSERT(DEFINED(ape_mz) ? ape_mz == IMAGE_BASE_VIRTUAL : 1, "linker panic");
ASSERT((DEFINED(__init_bss_end) ? __init_bss_end : 0) % __SIZEOF_POINTER__ == 0,
"__init_bss misalign");
@@ -656,6 +710,12 @@ ASSERT(DEFINED(_start) || DEFINED(_start16),
ASSERT(!DEFINED(_start16) || REAL(_end) < 65536,
"ape won't support non-tiny real mode programs");
ASSERT(IS2POW(ape_stack_memsz),
"ape_stack_memsz must be a two power");
ASSERT(!(ape_stack_vaddr & (ape_stack_memsz - 1)),
"ape_stack_vaddr must have ape_stack_memsz alignment; try using STATIC_STACK_ADDR(0x700000000000 - ape_stack_memsz);");
/* Let's not be like Knight Capital. */
/* NOCROSSREFS_TO(.test .text) */
+129 -27
View File
@@ -15,49 +15,151 @@
PKGS += APE
APE = $(APE_DEPS) \
$(APE_OBJS) \
APE = o/$(MODE)/ape/ape.o \
o/$(MODE)/ape/ape.lds
APE_BUILDSAFE = \
$(APE_DEPS) \
o/$(MODE)/ape/ape-buildsafe.o \
o/$(MODE)/ape/ape.lds
APE_NO_MODIFY_SELF = \
o/$(MODE)/ape/ape.lds \
o/$(MODE)/ape/ape-no-modify-self.o
APELINK = \
$(COMPILE) \
-ALINK.ape \
$(LINK) \
$(LINKARGS) \
APE_COPY_SELF = \
o/$(MODE)/ape/ape.lds \
o/$(MODE)/ape/ape-copy-self.o
APELINK = \
$(COMPILE) \
-ALINK.ape \
$(LINK) \
$(LINKARGS) \
$(OUTPUT_OPTION)
APE_LOADER_FLAGS = \
-DNDEBUG \
-iquote. \
-Wall \
-Wextra \
-fpie \
-Os \
-ffreestanding \
-mgeneral-regs-only \
-mno-red-zone \
-fno-ident \
-fno-gnu-unique \
-c \
$(OUTPUT_OPTION) \
$<
APE_FILES := $(wildcard ape/*.*)
APE_HDRS = $(filter %.h,$(APE_FILES))
APE_INCS = $(filter %.inc,$(APE_FILES))
APE_SRCS = $(filter %.S,$(APE_FILES))
APE_OBJS = $(APE_SRCS:%.S=o/$(MODE)/%.o)
APE_SRCS_C = ape/loader.c
APE_SRCS_S = $(filter %.S,$(APE_FILES))
APE_SRCS = $(APE_SRCS_C) $(APE_SRCS_S)
APE_OBJS = $(APE_SRCS_S:%.S=o/$(MODE)/%.o)
APE_CHECKS = $(APE_HDRS:%=o/%.ok)
o/$(MODE)/ape/ape.lds: \
ape/ape.lds \
ape/macros.internal.h \
libc/dce.h \
o/$(MODE)/ape/ape.lds: \
ape/ape.lds \
ape/macros.internal.h \
ape/relocations.h \
libc/intrin/bits.h \
libc/calls/struct/timespec.h \
libc/dce.h \
libc/elf/def.h \
libc/elf/pf2prot.internal.h \
libc/macros.internal.h \
libc/nt/pedef.internal.h \
libc/str/str.h \
libc/zip.h
o/ape/idata.inc: \
ape/idata.internal.h \
o/ape/idata.inc: \
ape/idata.internal.h \
ape/relocations.h
$(APE_OBJS): $(BUILD_FILES) \
ape/ape.mk
o/$(MODE)/ape/ape-no-modify-self.o: \
ape/ape.S \
ape/macros.internal.h \
ape/notice.inc \
ape/relocations.h \
libc/dce.h \
libc/elf/def.h \
libc/intrin/asancodes.h \
libc/macho.internal.h \
libc/macros.internal.h \
libc/nexgen32e/uart.internal.h \
libc/nexgen32e/vidya.internal.h \
libc/nt/pedef.internal.h \
libc/runtime/e820.internal.h \
libc/runtime/mman.internal.h \
libc/runtime/pc.internal.h \
libc/runtime/symbolic.h \
libc/sysv/consts/prot.h \
o/$(MODE)/ape/ape.elf
@$(COMPILE) \
-AOBJECTIFY.S \
$(OBJECTIFY.S) \
$(OUTPUT_OPTION) \
-DAPE_NO_MODIFY_SELF \
-DAPE_LOADER='"o/$(MODE)/ape/ape.elf"' $<
o/$(MODE)/ape/ape-no-modify-self.o: ape/ape.S
@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -DAPE_NO_MODIFY_SELF $<
o/$(MODE)/ape/ape-copy-self.o: \
ape/ape.S \
ape/macros.internal.h \
ape/notice.inc \
ape/relocations.h \
libc/dce.h \
libc/elf/def.h \
libc/intrin/asancodes.h \
libc/macho.internal.h \
libc/macros.internal.h \
libc/nexgen32e/uart.internal.h \
libc/nexgen32e/vidya.internal.h \
libc/nt/pedef.internal.h \
libc/runtime/e820.internal.h \
libc/runtime/mman.internal.h \
libc/runtime/pc.internal.h \
libc/runtime/symbolic.h \
libc/sysv/consts/prot.h
@$(COMPILE) \
-AOBJECTIFY.S \
$(OBJECTIFY.S) \
$(OUTPUT_OPTION) \
-DAPE_NO_MODIFY_SELF $<
o/$(MODE)/ape/ape-buildsafe.o: ape/ape.S
@$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) -DAPE_BUILDSAFE $<
o/$(MODE)/ape/loader.o: ape/loader.c ape/loader.h
@$(COMPILE) -AOBJECTIFY.c $(CC) -DSUPPORT_VECTOR=0b01111001 -g $(APE_LOADER_FLAGS)
o/$(MODE)/ape/loader-gcc.asm: ape/loader.c
@$(COMPILE) -AOBJECTIFY.c $(CC) -DSUPPORT_VECTOR=0b01111001 -S -g0 $(APE_LOADER_FLAGS)
o/$(MODE)/ape/loader-clang.asm: ape/loader.c
@$(COMPILE) -AOBJECTIFY.c $(CLANG) -DSUPPORT_VECTOR=0b01111001 -S -g0 $(APE_LOADER_FLAGS)
o/$(MODE)/ape/loader-xnu.o: ape/loader.c ape/loader.h
@$(COMPILE) -AOBJECTIFY.c $(CC) -DSUPPORT_VECTOR=0b00001000 -g $(APE_LOADER_FLAGS)
o/$(MODE)/ape/loader-xnu-gcc.asm: ape/loader.c
@$(COMPILE) -AOBJECTIFY.c $(CC) -DSUPPORT_VECTOR=0b00001000 -S -g0 $(APE_LOADER_FLAGS)
o/$(MODE)/ape/loader-xnu-clang.asm: ape/loader.c
@$(COMPILE) -AOBJECTIFY.c $(CLANG) -DSUPPORT_VECTOR=0b00001000 -S -g0 $(APE_LOADER_FLAGS)
o/$(MODE)/ape/ape.elf: o/$(MODE)/ape/ape.elf.dbg
o/$(MODE)/ape/ape.macho: o/$(MODE)/ape/ape.macho.dbg
o/$(MODE)/ape/ape.elf.dbg: \
o/$(MODE)/ape/loader.o \
o/$(MODE)/ape/loader-elf.o \
ape/loader.lds
@$(ELFLINK) -z max-page-size=0x10
o/$(MODE)/ape/ape.macho.dbg: \
o/$(MODE)/ape/loader-xnu.o \
o/$(MODE)/ape/loader-macho.o \
ape/loader-macho.lds
@$(ELFLINK) -z max-page-size=0x10
.PHONY: o/$(MODE)/ape
o/$(MODE)/ape: $(APE) \
$(APE_CHECKS) \
o/$(MODE)/ape: $(APE_CHECKS) \
o/$(MODE)/ape/ape.o \
o/$(MODE)/ape/ape.lds \
o/$(MODE)/ape/ape.elf \
o/$(MODE)/ape/ape.macho \
o/$(MODE)/ape/ape-copy-self.o \
o/$(MODE)/ape/ape-no-modify-self.o
+126
View File
@@ -0,0 +1,126 @@
#!/bin/sh
if [ $UID -eq 0 ]; then
SUDO=
else
SUDO=sudo
fi
echo "Actually Portable Executable (APE) Installer" >&2
echo "Author: Justine Tunney <[email protected]>" >&2
################################################################################
# INSTALL APE LOADER SYSTEMWIDE
if [ -f o/depend ]; then
# mkdeps.com build was successfully run so assume we can build
echo >&2
echo "recompiling ape loader" >&2
echo "running: make -j8 o//ape" >&2
make -j8 o//ape || exit
echo "done" >&2
elif [ -d build/bootstrap ]; then
# if make isn't being used then it's unlikely the user changed the sources
# in that case the prebuilt binaries should be completely up-to-date
echo "using prebuilt ape loader from cosmo repo" >&2
mkdir -p o//ape || exit
cp -af build/bootstrap/ape.elf o//ape/ape.elf || exit
cp -af build/bootstrap/ape.macho o//ape/ape.macho || exit
else
echo "no cosmopolitan libc repository here" >&2
echo "fetching ape loader from justine.lol" >&2
mkdir -p o//ape || exit
if command -v wget >/dev/null 2>&1; then
wget -qO o//ape/ape.elf https://justine.lol/ape.elf || exit
wget -qO o//ape/ape.macho https://justine.lol/ape.macho || exit
else
curl -Rso o//ape/ape.elf https://justine.lol/ape.elf || exit
curl -Rso o//ape/ape.macho https://justine.lol/ape.macho || exit
fi
chmod +x o//ape/ape.elf || exit
chmod +x o//ape/ape.macho || exit
fi
if [ "$(uname -s)" = "Darwin" ]; then
if ! [ /usr/bin/ape -nt o//ape/ape.macho ]; then
echo >&2
echo "installing o//ape/ape.elf to /usr/bin/ape" >&2
echo "$SUDO mv -f o//ape/ape.elf /usr/bin/ape" >&2
$SUDO cp -f o//ape/ape.macho /usr/bin/ape || exit
echo "done" >&2
fi
else
if ! [ /usr/bin/ape -nt o//ape/ape.elf ]; then
echo >&2
echo "installing o//ape/ape.elf to /usr/bin/ape" >&2
echo "$SUDO mv -f o//ape/ape.elf /usr/bin/ape" >&2
$SUDO cp -f o//ape/ape.elf /usr/bin/ape || exit
echo "done" >&2
fi
fi
################################################################################
# REGISTER APE LOADER WITH BINFMT_MISC TOO (LINUX-ONLY)
if [ x"$(uname -s)" = xLinux ]; then
if [ -e /proc/sys/fs/binfmt_misc/APE ]; then
echo >&2
echo it looks like APE is already registered with binfmt_misc >&2
echo To reinstall please run ape/apeuninstall.sh first >&2
echo please check that it is mapped to ape not /bin/sh >&2
echo cat /proc/sys/fs/binfmt_misc/APE >&2
cat /proc/sys/fs/binfmt_misc/APE >&2
exit
fi
if ! [ -e /proc/sys/fs/binfmt_misc ]; then
echo >&2
echo loading binfmt_misc into your kernel >&2
echo you may need to edit configs to persist across reboot >&2
echo $SUDO modprobe binfmt_misc >&2
$SUDO modprobe binfmt_misc || exit
echo done >&2
fi
if ! [ -e /proc/sys/fs/binfmt_misc/register ]; then
echo >&2
echo mounting binfmt_misc into your kernel >&2
echo you may need to edit configs to persist across reboot >&2
echo $SUDO mount -t binfmt_misc none /proc/sys/fs/binfmt_misc >&2
$SUDO mount -t binfmt_misc none /proc/sys/fs/binfmt_misc || exit
echo done >&2
fi
echo >&2
echo registering APE with binfmt_misc >&2
echo you may need to edit configs to persist across reboot >&2
echo '$SUDO sh -c "echo '"'"':APE:M::MZqFpD::/usr/bin/ape:'"'"' >/proc/sys/fs/binfmt_misc/register"' >&2
$SUDO sh -c "echo ':APE:M::MZqFpD::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register" || exit
echo done >&2
if [ x"$(cat /proc/sys/fs/binfmt_misc/status)" = xdisabled ]; then
echo >&2
echo enabling binfmt_misc >&2
echo you may need to edit configs to persist across reboot >&2
echo $SUDO sh -c 'echo 1 >/proc/sys/fs/binfmt_misc/status' >&2
$SUDO sh -c 'echo 1 >/proc/sys/fs/binfmt_misc/status' || exit
echo done >&2
fi
fi
################################################################################
{
echo
echo "------------------------------------------------------------------"
echo
echo "APE INSTALL COMPLETE"
echo
echo "If you decide to uninstall APE later on"
echo "you may do so using ape/apeuninstall.sh"
echo
echo "Enjoy your APE loader (>'.')>"
echo
} >&2
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
if [ "$UID" = "0" ]; then
SUDO=
else
SUDO=sudo
fi
{
echo
echo "APE Uninstaller intends to run"
echo
echo " $SUDO sh -c 'echo -1 >/proc/sys/fs/binfmt_misc/APE'"
echo " $SUDO rm -f /usr/bin/ape ~/.ape o/tmp/.ape /tmp/.ape"
echo
echo "You may then use ape/apeinstall.sh to reinstall it"
echo
} >&2
set -ex
if [ -f /proc/sys/fs/binfmt_misc/APE ]; then
$SUDO sh -c 'echo -1 >/proc/sys/fs/binfmt_misc/APE' || exit
fi
$SUDO rm -f /usr/bin/ape ~/.ape o/tmp/.ape o/tmp/ape /tmp/.ape /tmp/ape || exit
+10
View File
@@ -0,0 +1,10 @@
#ifndef COSMOPOLITAN_APE_CONFIG_H_
#define COSMOPOLITAN_APE_CONFIG_H_
#if !(__ASSEMBLER__ + __LINKER__ + 0)
#define STATIC_SYMBOL(NAME, VALUE) \
asm(".equ\t" NAME "," VALUE "\n\t" \
".globl\t" NAME)
#endif /* !(__ASSEMBLER__ + __LINKER__ + 0) */
#endif /* COSMOPOLITAN_APE_CONFIG_H_ */
+249
View File
@@ -0,0 +1,249 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/elf/def.h"
#include "libc/sysv/consts/prot.h"
#include "libc/macho.internal.h"
#include "libc/dce.h"
#include "libc/macros.internal.h"
// APE Loader Executable Structure
// Linux, FreeBSD, NetBSD, OpenBSD
.align 8
ehdr: .ascii "\177ELF"
.byte ELFCLASS64
.byte ELFDATA2LSB
.byte 1
.byte ELFOSABI_FREEBSD
.quad 0
.word ET_EXEC # e_type
.word EM_NEXGEN32E # e_machine
.long 1 # e_version
.quad _start # e_entry
.quad phdrs - ehdr # e_phoff
.quad 0 # e_shoff
.long 0 # e_flags
.word 64 # e_ehsize
.word 56 # e_phentsize
.word 4 # e_phnum
.word 0 # e_shentsize
.word 0 # e_shnum
.word 0 # e_shstrndx
.endobj ehdr,globl
.align 8
phdrs: .long PT_LOAD # p_type
.long PF_R|PF_X # p_flags
.quad 0 # p_offset
.quad ehdr # p_vaddr
.quad ehdr # p_paddr
.quad filesz # p_filesz
.quad filesz # p_memsz
.quad PAGESIZE # p_align
.long PT_LOAD # p_type
.long PF_R|PF_W # p_flags
.quad 0 # p_offset
.quad bss # p_vaddr
.quad bss # p_paddr
.quad 0 # p_filesz
.quad bsssize # p_memsz
.quad PAGESIZE # p_align
.long PT_GNU_STACK # p_type
.long PF_R|PF_W # p_flags
.quad 0 # p_offset
.quad 0 # p_vaddr
.quad 0 # p_paddr
.quad 0 # p_filesz
.quad 0 # p_memsz
.quad 16 # p_align
.long PT_NOTE # p_type
.long PF_R # p_flags
.quad note - ehdr # p_offset
.quad note # p_vaddr
.quad note # p_paddr
.quad notesize # p_filesz
.quad notesize # p_memsz
.quad 8 # p_align
.endobj phdrs
note: .long 2f-1f
.long 4f-3f
.long 1
1: .asciz "OpenBSD"
2: .align 4
3: .long 0
4: .long 2f-1f
.long 4f-3f
.long 1
1: .asciz "NetBSD"
2: .align 4
3: .long 901000000
4: .endobj note
notesize = . - note
.align 64,0 # for ape.S dd
.org 0x180 # for ape.S dd
// APE Loader XNU Header
//
// This header is dd'd backwards by the APE shell script when
// running on Mac OS X.
//
// @see ape/ape.S
macho: .long 0xFEEDFACE+1
.long MAC_CPU_NEXGEN32E
.long MAC_CPU_NEXGEN32E_ALL
.long MAC_EXECUTE
.long 5 # number of load commands
.long 60f-10f # size of all load commands
.long MAC_NOUNDEFS # flags
.long 0 # reserved
10: .long MAC_LC_SEGMENT_64
.long 20f-10b # unmaps first page dir
.ascin "__PAGEZERO",16 # consistent with linux
.quad 0,0x200000,0,0 # which forbids mem <2m
.long 0,0,0,0
20: .long MAC_LC_SEGMENT_64
.long 30f-20b
.ascin "__TEXT",16
.quad ehdr # vaddr
.quad 4096 # memsz
.quad 0 # file offset
.quad filesz # file size
.long PROT_EXEC|PROT_READ|PROT_WRITE # maxprot
.long PROT_EXEC|PROT_READ # initprot
.long 1 # segment section count
.long 0 # flags
210: .ascin "__text",16 # section name (.text)
.ascin "__TEXT",16
.quad _start # vaddr
.quad textsz # memsz
.long textoff # offset
.long 6 # align 2**6 = 64
.long 0 # reloc table offset
.long 0 # relocation count
.long MAC_S_ATTR_SOME_INSTRUCTIONS # section type & attributes
.long 0,0,0 # reserved
30: .long MAC_LC_SEGMENT_64
.long 40f-30b
.ascin "__DATA",16
.quad bss # vaddr
.quad bsssize # memsz
.quad 0 # offset
.quad 0 # file size
.long PROT_EXEC|PROT_READ|PROT_WRITE # maxprot
.long PROT_READ|PROT_WRITE # initprot
.long 1 # segment section count
.long 0 # flags
310: .ascin "__bss",16 # section name (.bss)
.ascin "__DATA",16
.quad bss # vaddr
.quad bsssize # memsz
.long 0 # offset
.long 12 # align 2**12 = 4096
.long 0 # reloc table offset
.long 0 # relocation count
.long MAC_S_ZEROFILL # section type & attributes
.long 0,0,0 # reserved
40: .long MAC_LC_UUID
.long 50f-40b
.quad 0x3fb29ee4ac6c87aa # uuid1
.quad 0xdd2c9bb866d9eef8 # uuid2
50: .long MAC_LC_UNIXTHREAD
.long 60f-50b # cmdsize
.long MAC_THREAD_NEXGEN32E # flavaflav
.long (520f-510f)/4 # count
510: .quad 0 # rax
.quad 0 # rbx
.quad 0 # rcx
.quad XNU # rdx
.quad 0 # rdi
.quad 0 # rsi
.quad 0 # rbp
.quad 0 # rsp
.quad 0 # r8
.quad 0 # r9
.quad 0 # r10
.quad 0 # r11
.quad 0 # r12
.quad 0 # r13
.quad 0 # r14
.quad 0 # r15
.quad _start # rip
.quad 0 # rflags
.quad 0 # cs
.quad 0 # fs
.quad 0 # gs
520:
60:
.endobj macho
.align 64,0 # for ape.S dd
.org 0x400 # for ape.S dd
// Ape Loader Entrpoint
//
// This is normally called by the operating system. However it may
// be called by the Actually Portable Executables themselves, when
// re-executing a program. Just do this:
//
// memcpy(0x200000, loader)
// lea handoff(%rip),%rcx
// lea argblock(%rip),%rsp
// jmp 0x200400
//
// @see APE_LOADER_ENTRY
// @see ape/loader.h
_start:
// Hack for detecting M1 Rosetta environment.
// https://github.com/jart/cosmopolitan/issues/429#issuecomment-1166704377
cmp $-1,%ebx
jne 0f
cmp $+1,%edx
jne 0f
mov $XNU,%dl
xor %ecx,%ecx
0: mov %rsp,%rsi
jmp ApeLoader
.endfn _start,globl
// System Call Entrpoint
//
// This function is used by the APE loader to make system calls.
// We also pass a reference to this function to the APE binary's
// _start() function. It's needed because on OpenBSD, msyscall()
// restricts which pages can issue system calls, and it can only
// be called once. Therefore if we want to be load and re-load a
// binary multiple times without calling the system execve(), we
// need to be able to handover the SYSCALL function. We hardcode
// this to a fixed address, but that shouldn't be used, since we
// would ideally want to move it to a random page in the future.
__syscall_loader:
clc
syscall
jc 1f
ret
1: neg %rax
ret
.endfn __syscall_loader,globl
+138
View File
@@ -0,0 +1,138 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macho.internal.h"
#include "libc/sysv/consts/prot.h"
#include "libc/dce.h"
#include "libc/macros.internal.h"
// APE Loader Executable Structure for XNU
.align 4096
macho: .long 0xFEEDFACE+1
.long MAC_CPU_NEXGEN32E
.long MAC_CPU_NEXGEN32E_ALL
.long MAC_EXECUTE
.long 5 # number of load commands
.long 60f-10f # size of all load commands
.long MAC_NOUNDEFS # flags
.long 0 # reserved
10: .long MAC_LC_SEGMENT_64
.long 20f-10b # unmaps first page dir
.ascin "__PAGEZERO",16 # consistent with linux
.quad 0,0x200000,0,0 # which forbids mem <2m
.long 0,0,0,0
20: .long MAC_LC_SEGMENT_64
.long 30f-20b
.ascin "__TEXT",16
.quad macho # vaddr
.quad 4096 # memsz
.quad 0 # file offset
.quad filesz # file size
.long PROT_EXEC|PROT_READ|PROT_WRITE # maxprot
.long PROT_EXEC|PROT_READ # initprot
.long 1 # segment section count
.long 0 # flags
210: .ascin "__text",16 # section name (.text)
.ascin "__TEXT",16
.quad _start # vaddr
.quad textsz # memsz
.long textoff # offset
.long 6 # align 2**3 = 64
.long 0 # reloc table offset
.long 0 # relocation count
.long MAC_S_ATTR_SOME_INSTRUCTIONS # section type & attributes
.long 0,0,0 # reserved
30: .long MAC_LC_SEGMENT_64
.long 40f-30b
.ascin "__DATA",16
.quad bss # vaddr
.quad bsssize # memsz
.quad 0 # offset
.quad 0 # file size
.long PROT_EXEC|PROT_READ|PROT_WRITE # maxprot
.long PROT_READ|PROT_WRITE # initprot
.long 1 # segment section count
.long 0 # flags
310: .ascin "__bss",16 # section name (.bss)
.ascin "__DATA",16
.quad bss # vaddr
.quad bsssize # memsz
.long 0 # offset
.long 12 # align 2**12 = 4096
.long 0 # reloc table offset
.long 0 # relocation count
.long MAC_S_ZEROFILL # section type & attributes
.long 0,0,0 # reserved
40: .long MAC_LC_UUID
.long 50f-40b
.quad 0x3fb29ee4ac6c87aa # uuid1
.quad 0xdd2c9bb866d9eef8 # uuid2
50: .long MAC_LC_UNIXTHREAD
.long 60f-50b # cmdsize
.long MAC_THREAD_NEXGEN32E # flavaflav
.long (520f-510f)/4 # count
510: .quad 0 # rax
.quad 0 # rbx
.quad 0 # rcx
.quad XNU # rdx
.quad 0 # rdi
.quad 0 # rsi
.quad 0 # rbp
.quad 0 # rsp
.quad 0 # r8
.quad 0 # r9
.quad 0 # r10
.quad 0 # r11
.quad 0 # r12
.quad 0 # r13
.quad 0 # r14
.quad 0 # r15
.quad _start # rip
.quad 0 # rflags
.quad 0 # cs
.quad 0 # fs
.quad 0 # gs
520:
60:
.endobj macho,globl
.align 64
_start:
// Hack for detecting M1 Rosetta environment.
// https://github.com/jart/cosmopolitan/issues/429#issuecomment-1166704377
cmp $-1,%ebx
jne 0f
cmp $+1,%edx
jne 0f
mov $XNU,%dl
xor %ecx,%ecx
0: mov %rsp,%rsi
jmp ApeLoader
.endfn _start,globl
__syscall_loader:
clc
syscall
jc 1f
ret
1: neg %rax
ret
.endfn __syscall_loader,globl
+42
View File
@@ -0,0 +1,42 @@
/*-*- mode: ld-script; indent-tabs-mode: nil; tab-width: 2; coding: utf-8 -*-│
│vi: set et sts=2 tw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
ENTRY(_start)
SECTIONS {
. = 0x200000;
.text : {
*(.text)
*(.rodata .rodata.*)
. = ALIGN(4096);
}
filesz = . - macho;
textsz = . - _start;
.bss ALIGN(4096) : {
bss = .;
*(.bss)
. = ALIGN(4096);
}
memsz = . - macho;
/DISCARD/ : {
*(.*)
}
}
bsssize = SIZEOF(.bss);
textoff = _start - macho;
+791
View File
@@ -0,0 +1,791 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "ape/loader.h"
#define SET_EXE_FILE 0 /* needs root ;_; */
#define TROUBLESHOOT 0
#define TROUBLESHOOT_OS LINUX
/**
* @fileoverview APE Loader for GNU/Systemd/XNU/FreeBSD/NetBSD/OpenBSD
*
* We recommend using the normal APE design, where binaries assimilate
* themselves once by self-modifying the first 64 bytes. If that can't
* meet your requirements then we provide an excellent alternative.
*
* m=tiny
* make -j8 MODE=$m o/$m/ape o/$m/examples/printargs.com
* o/$m/ape/ape.elf o/$m/examples/printargs.com
*
* This is an embeddable Actually Portable Executable interpreter. The
* `ape/ape.S` bootloader embeds this binary inside each binary that's
* linked using `$(APE_NO_MODIFY_SELF)` so it is an automated seamless
* process. the shell script at the top of the .COM files will copy it
* to `${TMPDIR:-${HOME:-.}}/.ape` and call execve(). It's a zero copy
* operation in praxis since this payload uses mmap() to load the rest
* of your executable the same way the kernel does, based on ELF phdrs
* which are located in accordance with the first sh printf statement.
*
* APE executables will look for this program on the system path first
* so your APE loader may be installed to your system as follows:
*
* m=tiny
* make -j8 MODE=$m o/$m/ape/ape.elf
* sudo cp o/$m/ape/ape.elf /usr/bin/ape
*
* For Mac OS X systems you should install the `ape.macho` executable:
*
* sudo cp o/$m/ape/ape.macho /usr/bin/ape
*
* Your APE loader may be used as a shebang interpreter by doing this:
*
* #!/usr/bin/ape python.com
* # -*- python -*-
* print("hello world")
*
* However you won't need to do that, if your APE Loader is registered
* as a binfmt_misc interpreter. You can do that as follows with root:
*
* sudo cp -f o/$m/ape/ape.elf /usr/bin
* f=/proc/sys/fs/binfmt_misc/register
* sudo sh -c "echo ':APE:M::MZqFpD::/usr/bin/ape:' >$f"
*
* If the register file doesn't exist on your Linux machine then you'd
* load it using the following commands:
*
* sudo modprobe binfmt_misc
* sudo mount -t binfmt_misc none /proc/sys/fs/binfmt_misc
*
* You should now experience a performance boost, and you can also now
* use the regular shebang form:
*
* #!/usr/bin/python.com
* # -*- python -*-
* print("hello world")
*
* @note this can probably be used as a binfmt_misc interpreter
*/
#define LINUX 1
#define XNU 8
#define OPENBSD 16
#define FREEBSD 32
#define NETBSD 64
#define SupportsLinux() (SUPPORT_VECTOR & LINUX)
#define SupportsXnu() (SUPPORT_VECTOR & XNU)
#define SupportsFreebsd() (SUPPORT_VECTOR & FREEBSD)
#define SupportsOpenbsd() (SUPPORT_VECTOR & OPENBSD)
#define SupportsNetbsd() (SUPPORT_VECTOR & NETBSD)
#define IsLinux() (SupportsLinux() && os == LINUX)
#define IsXnu() (SupportsXnu() && os == XNU)
#define IsFreebsd() (SupportsFreebsd() && os == FREEBSD)
#define IsOpenbsd() (SupportsOpenbsd() && os == OPENBSD)
#define IsNetbsd() (SupportsNetbsd() && os == NETBSD)
#define O_RDONLY 0
#define PROT_READ 1
#define PROT_WRITE 2
#define PROT_EXEC 4
#define MAP_SHARED 1
#define MAP_PRIVATE 2
#define MAP_FIXED 16
#define MAP_ANONYMOUS (IsLinux() ? 32 : 4096)
#define AT_EXECFN_LINUX 31
#define AT_EXECFN_NETBSD 2014
#define ELFCLASS64 2
#define ELFDATA2LSB 1
#define EM_NEXGEN32E 62
#define ET_EXEC 2
#define PT_LOAD 1
#define PT_DYNAMIC 2
#define EI_CLASS 4
#define EI_DATA 5
#define PF_X 1
#define PF_W 2
#define PF_R 4
#define X_OK 1
#define XCR0_SSE 2
#define XCR0_AVX 4
#define PR_SET_MM 35
#define PR_SET_MM_EXE_FILE 13
#define Read32(S) \
((unsigned)(255 & (S)[3]) << 030 | (unsigned)(255 & (S)[2]) << 020 | \
(unsigned)(255 & (S)[1]) << 010 | (unsigned)(255 & (S)[0]) << 000)
#define Read64(S) \
((unsigned long)(255 & (S)[7]) << 070 | \
(unsigned long)(255 & (S)[6]) << 060 | \
(unsigned long)(255 & (S)[5]) << 050 | \
(unsigned long)(255 & (S)[4]) << 040 | \
(unsigned long)(255 & (S)[3]) << 030 | \
(unsigned long)(255 & (S)[2]) << 020 | \
(unsigned long)(255 & (S)[1]) << 010 | \
(unsigned long)(255 & (S)[0]) << 000)
struct PathSearcher {
char os;
unsigned long namelen;
const char *name;
const char *syspath;
char path[1024];
};
struct ElfEhdr {
unsigned char e_ident[16];
unsigned short e_type;
unsigned short e_machine;
unsigned e_version;
unsigned long e_entry;
unsigned long e_phoff;
unsigned long e_shoff;
unsigned e_flags;
unsigned short e_ehsize;
unsigned short e_phentsize;
unsigned short e_phnum;
unsigned short e_shentsize;
unsigned short e_shnum;
unsigned short e_shstrndx;
};
struct ElfPhdr {
unsigned p_type;
unsigned p_flags;
unsigned long p_offset;
unsigned long p_vaddr;
unsigned long p_paddr;
unsigned long p_filesz;
unsigned long p_memsz;
unsigned long p_align;
};
extern char ehdr[];
extern char _end[];
static void *syscall;
static char relocated;
static struct PathSearcher ps;
extern char __syscall_loader[];
static int ToLower(int c) {
return 'A' <= c && c <= 'Z' ? c + ('a' - 'A') : c;
}
char *MemCpy(char *d, const char *s, unsigned long n) {
unsigned long i = 0;
for (; i < n; ++i) d[i] = s[i];
return d + n;
}
static unsigned long StrLen(const char *s) {
unsigned long n = 0;
while (*s++) ++n;
return n;
}
static const char *MemChr(const char *s, unsigned char c, unsigned long n) {
for (; n; --n, ++s) {
if ((*s & 255) == c) {
return s;
}
}
return 0;
}
static char *GetEnv(char **p, const char *s) {
unsigned long i, j;
if (p) {
for (i = 0; p[i]; ++i) {
for (j = 0;; ++j) {
if (!s[j]) {
if (p[i][j] == '=') {
return p[i] + j + 1;
}
break;
}
if (s[j] != p[i][j]) {
break;
}
}
}
}
return 0;
}
static char *Utoa(char p[21], unsigned long x) {
char t;
unsigned long i, a, b;
i = 0;
do {
p[i++] = x % 10 + '0';
x = x / 10;
} while (x > 0);
p[i] = '\0';
if (i) {
for (a = 0, b = i - 1; a < b; ++a, --b) {
t = p[a];
p[a] = p[b];
p[b] = t;
}
}
return p + i;
}
static char *Itoa(char p[21], long x) {
if (x < 0) *p++ = '-', x = -(unsigned long)x;
return Utoa(p, x);
}
#if TROUBLESHOOT
const char *DescribeOs(int os) {
if (IsLinux()) {
return "GNU/SYSTEMD";
} else if (IsXnu()) {
return "XNU";
} else if (IsFreebsd()) {
return "FREEBSD";
} else if (IsOpenbsd()) {
return "OPENBSD";
} else if (IsNetbsd()) {
return "NETBSD";
} else {
return "WUT";
}
}
#endif
__attribute__((__noreturn__)) static void Exit(int rc, int os) {
asm volatile("call\t*%2"
: /* no outputs */
: "a"((IsLinux() ? 60 : 1) | (IsXnu() ? 0x2000000 : 0)), "D"(rc),
"rm"(syscall)
: "memory");
__builtin_unreachable();
}
static void Close(int fd, int os) {
int ax, di;
asm volatile("call\t*%4"
: "=a"(ax), "=D"(di)
: "0"((IsLinux() ? 3 : 6) | (IsXnu() ? 0x2000000 : 0)), "1"(fd),
"rm"(syscall)
: "rcx", "rdx", "rsi", "r8", "r9", "r10", "r11", "memory", "cc");
}
static int Read(int fd, void *data, int size, int os) {
long si;
int ax, di, dx;
asm volatile("call\t*%8"
: "=a"(ax), "=D"(di), "=S"(si), "=d"(dx)
: "0"((IsLinux() ? 0 : 3) | (IsXnu() ? 0x2000000 : 0)), "1"(fd),
"2"(data), "3"(size), "rm"(syscall)
: "rcx", "r8", "r9", "r10", "r11", "memory");
return ax;
}
static void Write(int fd, const void *data, int size, int os) {
long si;
int ax, di, dx;
asm volatile("call\t*%8"
: "=a"(ax), "=D"(di), "=S"(si), "=d"(dx)
: "0"((IsLinux() ? 1 : 4) | (IsXnu() ? 0x2000000 : 0)), "1"(fd),
"2"(data), "3"(size), "rm"(syscall)
: "rcx", "r8", "r9", "r10", "r11", "memory", "cc");
}
static void Execve(const char *prog, char **argv, char **envp, int os) {
long ax, di, si, dx;
asm volatile("call\t*%8"
: "=a"(ax), "=D"(di), "=S"(si), "=d"(dx)
: "0"(59 | (IsXnu() ? 0x2000000 : 0)), "1"(prog), "2"(argv),
"3"(envp), "rm"(syscall)
: "rcx", "r8", "r9", "r10", "r11", "memory", "cc");
}
static int Access(const char *path, int mode, int os) {
int ax, si;
long dx, di;
asm volatile("call\t*%7"
: "=a"(ax), "=D"(di), "=S"(si), "=d"(dx)
: "0"((IsLinux() ? 21 : 33) | (IsXnu() ? 0x2000000 : 0)),
"1"(path), "2"(mode), "rm"(syscall)
: "rcx", "r8", "r9", "r10", "r11", "memory", "cc");
return ax;
}
static int Msyscall(long p, long n, int os) {
int ax;
long di, si;
if (!IsOpenbsd()) {
return 0;
} else {
asm volatile("call\t*%6"
: "=a"(ax), "=D"(di), "=S"(si)
: "0"(37), "1"(p), "2"(n), "rm"(syscall)
: "rcx", "rdx", "r8", "r9", "r10", "r11", "memory", "cc");
return ax;
}
}
static int Open(const char *path, int flags, int mode, int os) {
long di;
int ax, dx, si;
asm volatile("call\t*%8"
: "=a"(ax), "=D"(di), "=S"(si), "=d"(dx)
: "0"((IsLinux() ? 2 : 5) | (IsXnu() ? 0x2000000 : 0)),
"1"(path), "2"(flags), "3"(mode), "rm"(syscall)
: "rcx", "r8", "r9", "r10", "r11", "memory", "cc");
return ax;
}
__attribute__((__noinline__)) static long Mmap(long addr, long size, int prot,
int flags, int fd, long off,
int os) {
long ax, di, si, dx;
register int flags_ asm("r10") = flags;
register int fd_ asm("r8") = fd;
register long off_ asm("r9") = off;
asm volatile("push\t%%r9\n\t"
"push\t%%r9\n\t"
"call\t*%7\n\t"
"pop\t%%r9\n\t"
"pop\t%%r9"
: "=a"(ax), "=D"(di), "=S"(si), "=d"(dx), "+r"(flags_),
"+r"(fd_), "+r"(off_)
: "rm"(syscall),
"0"((IsLinux() ? 9
: IsFreebsd() ? 477
: 197) |
(IsXnu() ? 0x2000000 : 0)),
"1"(addr), "2"(size), "3"(prot)
: "rcx", "r11", "memory", "cc");
return ax;
}
int MunmapLinux(const void *addr, unsigned long size) {
int ax;
asm volatile("syscall"
: "=a"(ax)
: "0"(11), "D"(addr), "S"(size)
: "rcx", "r11", "memory");
return ax;
}
int PrctlLinux(int op, long a, long b, long c, long d) {
int rc;
asm volatile("mov\t%5,%%r10\n\t"
"mov\t%6,%%r8\n\t"
"syscall"
: "=a"(rc)
: "0"(157), "D"(op), "S"(a), "d"(b), "g"(c), "g"(d)
: "rcx", "r8", "r10", "r11", "memory");
return rc;
}
static void Emit(int os, const char *s) {
Write(2, s, StrLen(s), os);
}
static void Perror(int os, const char *c, int rc, const char *s) {
char ibuf[21];
Emit(os, "ape error: ");
Emit(os, c);
Emit(os, ": ");
Emit(os, s);
if (rc) {
Emit(os, " failed errno=");
Itoa(ibuf, -rc);
Emit(os, ibuf);
}
Emit(os, "\n");
}
__attribute__((__noreturn__)) static void Pexit(int os, const char *c, int rc,
const char *s) {
Perror(os, c, rc, s);
Exit(127, os);
}
static int StrCmp(const char *l, const char *r) {
unsigned long i = 0;
while (l[i] == r[i] && r[i]) ++i;
return (l[i] & 255) - (r[i] & 255);
}
static char EndsWithIgnoreCase(const char *p, unsigned long n, const char *s) {
unsigned long i, m;
if (n >= (m = StrLen(s))) {
for (i = n - m; i < n; ++i) {
if (ToLower(p[i]) != *s++) {
return 0;
}
}
return 1;
} else {
return 0;
}
}
static char IsComPath(struct PathSearcher *ps) {
return EndsWithIgnoreCase(ps->name, ps->namelen, ".com") ||
EndsWithIgnoreCase(ps->name, ps->namelen, ".exe") ||
EndsWithIgnoreCase(ps->name, ps->namelen, ".com.dbg");
}
static char AccessCommand(struct PathSearcher *ps, const char *suffix,
unsigned long pathlen) {
unsigned long suffixlen;
suffixlen = StrLen(suffix);
if (pathlen + 1 + ps->namelen + suffixlen + 1 > sizeof(ps->path)) return 0;
if (pathlen && ps->path[pathlen - 1] != '/') ps->path[pathlen++] = '/';
MemCpy(ps->path + pathlen, ps->name, ps->namelen);
MemCpy(ps->path + pathlen + ps->namelen, suffix, suffixlen + 1);
return !Access(ps->path, X_OK, ps->os);
}
static char SearchPath(struct PathSearcher *ps, const char *suffix) {
const char *p;
unsigned long i;
for (p = ps->syspath;;) {
for (i = 0; p[i] && p[i] != ':'; ++i) {
if (i < sizeof(ps->path)) {
ps->path[i] = p[i];
}
}
if (AccessCommand(ps, suffix, i)) {
return 1;
} else if (p[i] == ':') {
p += i + 1;
} else {
return 0;
}
}
}
static char FindCommand(struct PathSearcher *ps, const char *suffix) {
if (MemChr(ps->name, '/', ps->namelen) ||
MemChr(ps->name, '\\', ps->namelen)) {
ps->path[0] = 0;
return AccessCommand(ps, suffix, 0);
} else {
if (AccessCommand(ps, suffix, 0)) return 1;
}
return SearchPath(ps, suffix);
}
static char *Commandv(struct PathSearcher *ps, int os, const char *name,
const char *syspath) {
ps->os = os;
ps->syspath = syspath ? syspath : "/bin:/usr/local/bin:/usr/bin";
if (!(ps->namelen = StrLen((ps->name = name)))) return 0;
if (ps->namelen + 1 > sizeof(ps->path)) return 0;
if (FindCommand(ps, "") || (!IsComPath(ps) && FindCommand(ps, ".com"))) {
return ps->path;
} else {
return 0;
}
}
__attribute__((__noreturn__)) static void Spawn(int os, const char *exe, int fd,
long *sp, char *page,
struct ElfEhdr *e) {
long rc;
unsigned long i;
int prot, flags;
struct ElfPhdr *p;
long code, codesize;
if (e->e_type != ET_EXEC) {
Pexit(os, exe, 0, "ELF e_type != ET_EXEC");
}
if (e->e_machine != EM_NEXGEN32E) {
Pexit(os, exe, 0, "ELF e_machine != EM_NEXGEN32E");
}
if (e->e_ident[EI_CLASS] != ELFCLASS64) {
Pexit(os, exe, 0, "ELF e_ident[EI_CLASS] != ELFCLASS64");
}
if (e->e_ident[EI_DATA] != ELFDATA2LSB) {
Pexit(os, exe, 0, "ELF e_ident[EI_DATA] != ELFDATA2LSB");
}
if (e->e_phoff + e->e_phnum * sizeof(*p) > 0x1000) {
Pexit(os, exe, 0, "ELF phdrs need to be in first page");
}
code = 0;
codesize = 0;
for (p = (struct ElfPhdr *)(page + e->e_phoff), i = e->e_phnum; i--;) {
if (p[i].p_type == PT_DYNAMIC) {
Pexit(os, exe, 0, "not a real executable");
}
if (p[i].p_type != PT_LOAD) continue;
if ((p[i].p_vaddr | p[i].p_filesz | p[i].p_memsz | p[i].p_offset) & 0xfff) {
Pexit(os, exe, 0, "APE phdrs must be 4096-aligned and 4096-padded");
}
prot = 0;
flags = MAP_FIXED | MAP_PRIVATE;
if (p[i].p_flags & PF_R) {
prot |= PROT_READ;
}
if (p[i].p_flags & PF_W) {
prot |= PROT_WRITE;
}
if (p[i].p_flags & PF_X) {
prot |= PROT_EXEC;
code = p[i].p_vaddr;
codesize = p[i].p_filesz;
}
if (p[i].p_memsz > p[i].p_filesz) {
if ((rc = Mmap(p[i].p_vaddr + p[i].p_filesz, p[i].p_memsz - p[i].p_filesz,
prot, flags | MAP_ANONYMOUS, -1, 0, os)) < 0) {
Pexit(os, exe, rc, "bss mmap()");
}
}
if (p[i].p_filesz) {
if ((rc = Mmap(p[i].p_vaddr, p[i].p_filesz, prot, flags, fd,
p[i].p_offset, os)) < 0) {
Pexit(os, exe, rc, "image mmap()");
}
}
}
if (!code) {
Pexit(os, exe, 0, "ELF needs PT_LOAD phdr w/ PF_X");
}
#if SET_EXE_FILE
// change /proc/pid/exe to new executable path
if (IsLinux() && relocated) {
MunmapLinux((char *)0x200000, (long)(_end - ehdr));
PrctlLinux(PR_SET_MM, PR_SET_MM_EXE_FILE, fd, 0, 0);
}
#endif
if (relocated) {
int ax;
asm volatile("syscall"
: "=a"(ax)
: "0"(1), "D"(1), "S"("hello\n"), "d"(6)
: "rcx", "r11", "memory");
}
Close(fd, os);
// authorize only the loaded program to issue system calls. if this
// fails, then we pass a link to our syscall function to the program
// since it probably means a userspace program executed this loader
// and passed us a custom syscall function earlier.
if (Msyscall(code, codesize, os) != -1) {
syscall = 0;
}
#if TROUBLESHOOT
Emit(TROUBLESHOOT_OS, "preparing to jump\n");
#endif
// we clear all the general registers we can to have some wiggle room
// to extend the behavior of this loader in the future. we don't need
// to clear the xmm registers since the ape loader should be compiled
// with the -mgeneral-regs-only flag.
register void *r8 asm("r8") = syscall;
asm volatile("xor\t%%eax,%%eax\n\t"
"xor\t%%r9d,%%r9d\n\t"
"xor\t%%r10d,%%r10d\n\t"
"xor\t%%r11d,%%r11d\n\t"
"xor\t%%ebx,%%ebx\n\t" // netbsd doesnt't clear this
"xor\t%%r12d,%%r12d\n\t" // netbsd doesnt't clear this
"xor\t%%r13d,%%r13d\n\t" // netbsd doesnt't clear this
"xor\t%%r14d,%%r14d\n\t" // netbsd doesnt't clear this
"xor\t%%r15d,%%r15d\n\t" // netbsd doesnt't clear this
"mov\t%%rdx,%%rsp\n\t"
"xor\t%%edx,%%edx\n\t"
"push\t%%rsi\n\t"
"xor\t%%esi,%%esi\n\t"
"xor\t%%ebp,%%ebp\n\t"
"ret"
: /* no outputs */
: "D"(IsFreebsd() ? sp : 0), "S"(e->e_entry), "d"(sp), "c"(os),
"r"(r8)
: "memory");
__builtin_unreachable();
}
__attribute__((__noreturn__)) void ApeLoader(long di, long *sp, char dl,
struct ApeLoader *handoff) {
int rc;
long *auxv;
struct ElfEhdr *eh;
int c, i, fd, os, argc;
char *p, *exe, *prog, **argv, **envp, *page;
static union {
struct ElfEhdr ehdr;
char p[0x1000];
} u;
// detect freebsd
if (handoff) {
os = handoff->os;
} else if (SupportsXnu() && dl == XNU) {
os = XNU;
} else if (SupportsFreebsd() && di) {
os = FREEBSD;
sp = (long *)di;
} else {
os = 0;
}
// extract arguments
argc = *sp;
argv = (char **)(sp + 1);
envp = (char **)(sp + 1 + argc + 1);
auxv = (long *)(sp + 1 + argc + 1);
for (;;) {
if (!*auxv++) {
break;
}
}
// get syscall function pointer
if (handoff && handoff->syscall) {
syscall = handoff->syscall;
} else {
syscall = __syscall_loader;
}
if (handoff) {
// we were called by ape_execve()
// no argument parsing is needed
// no path searching is needed
exe = handoff->prog;
fd = handoff->fd;
exe = handoff->prog;
page = handoff->page;
eh = (struct ElfEhdr *)handoff->page;
} else {
// detect openbsd
if (SupportsOpenbsd() && !os && !auxv[0]) {
os = OPENBSD;
}
// detect netbsd
if (SupportsNetbsd() && !os) {
for (; auxv[0]; auxv += 2) {
if (auxv[0] == AT_EXECFN_NETBSD) {
os = NETBSD;
break;
}
}
}
// default operating system
if (!os) {
os = LINUX;
}
#if SET_EXE_FILE
if (IsLinux() && !relocated) {
char *b1 = (char *)0x200000;
char *b2 = (char *)0x300000;
void (*pApeLoader)(long, long *, char, struct ApeLoader *);
Mmap((long)b2, (long)(_end - ehdr), PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0, os);
relocated = 1;
MemCpy(b2, b1, (long)(_end - ehdr));
pApeLoader = (void *)((char *)&ApeLoader - b1 + b2);
pApeLoader(di, sp, dl, handoff);
}
#endif
// we can load via shell, shebang, or binfmt_misc
if (argc >= 3 && !StrCmp(argv[1], "-")) {
// if the first argument is a hyphen then we give the user the
// power to change argv[0] or omit it entirely. most operating
// systems don't permit the omission of argv[0] but we do, b/c
// it's specified by ANSI X3.159-1988.
prog = (char *)sp[3];
argc = sp[3] = sp[0] - 3;
argv = (char **)((sp += 3) + 1);
} else if (argc < 2) {
Emit(os, "usage: ape PROG [ARGV1,ARGV2,...]\n"
" ape - PROG [ARGV0,ARGV1,...]\n"
"αcτµαlly pδrταblε εxεcµταblε loader v1.o\n"
"copyright 2022 justine alexandra roberts tunney\n"
"https://justine.lol/ape.html\n");
Exit(1, os);
} else {
prog = (char *)sp[2];
argc = sp[1] = sp[0] - 1;
argv = (char **)((sp += 1) + 1);
}
if (!(exe = Commandv(&ps, os, prog, GetEnv(envp, "PATH")))) {
Pexit(os, prog, 0, "not found (maybe chmod +x)");
} else if ((fd = Open(exe, O_RDONLY, 0, os)) < 0) {
Pexit(os, exe, fd, "open");
} else if ((rc = Read(fd, u.p, sizeof(u.p), os)) < 0) {
Pexit(os, exe, rc, "read");
} else if (rc != sizeof(u.p) && Read32(u.p) != Read32("\177ELF")) {
Pexit(os, exe, 0, "too small");
}
page = u.p;
eh = &u.ehdr;
}
#if TROUBLESHOOT
Emit(TROUBLESHOOT_OS, "os = ");
Emit(TROUBLESHOOT_OS, DescribeOs(os));
Emit(TROUBLESHOOT_OS, "\n");
for (i = 0; i < argc; ++i) {
Emit(TROUBLESHOOT_OS, "argv = ");
Emit(TROUBLESHOOT_OS, argv[i]);
Emit(TROUBLESHOOT_OS, "\n");
}
#endif
if ((IsXnu() && Read32(page) == 0xFEEDFACE + 1) ||
(!IsXnu() && Read32(page) == Read32("\177ELF"))) {
Close(fd, os);
Execve(exe, argv, envp, os);
}
// TODO(jart): Parse Mach-O for old APE binary support on XNU.
for (p = page; p < page + sizeof(u.p); ++p) {
if (Read64(p) != Read64("printf '")) continue;
for (i = 0, p += 8; p + 3 < page + sizeof(u.p) && (c = *p++) != '\'';) {
if (c == '\\') {
if ('0' <= *p && *p <= '7') {
c = *p++ - '0';
if ('0' <= *p && *p <= '7') {
c *= 8;
c += *p++ - '0';
if ('0' <= *p && *p <= '7') {
c *= 8;
c += *p++ - '0';
}
}
}
}
page[i++] = c;
}
if (i >= 64 && Read32(page) == Read32("\177ELF")) {
Spawn(os, exe, fd, sp, page, eh);
}
}
Pexit(os, exe, 0, "could not find printf elf in first page");
}
+20
View File
@@ -0,0 +1,20 @@
#ifndef COSMOPOLITAN_APE_LOADER_H_
#define COSMOPOLITAN_APE_LOADER_H_
#define APE_LOADER_BASE 0x200000
#define APE_LOADER_SIZE 0x200000
#define APE_LOADER_ENTRY 0x200400
#define APE_LOADER_BSS (PAGESIZE * 2)
#define APE_LOADER_STACK 0x7f0000000000
#define APE_BLOCK_BASE 0x7e0000000000
#define APE_BLOCK_SIZE 0x000200000000
struct ApeLoader {
int fd;
int os;
char *prog;
char *page;
void *syscall;
};
#endif /* COSMOPOLITAN_APE_LOADER_H_ */
+43
View File
@@ -0,0 +1,43 @@
/*-*- mode: ld-script; indent-tabs-mode: nil; tab-width: 2; coding: utf-8 -*-│
│vi: set et sts=2 tw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
ENTRY(_start)
SECTIONS {
. = 0x200000;
.text : {
*(.text)
*(.rodata .rodata.*)
. = ALIGN(64);
}
filesz = . - ehdr;
textsz = . - _start;
.bss ALIGN(4096) : {
bss = .;
*(.bss)
. = ALIGN(4096);
}
_end = .;
memsz = _end - ehdr;
/DISCARD/ : {
*(.*)
}
}
bsssize = SIZEOF(.bss);
textoff = _start - ehdr;
+1 -1
View File
@@ -54,7 +54,7 @@
jmp 297f
.byte 0xbf # mov $0x????xxxx,%edi
.long \address
297:
297:
.endm
// Loads 16-bit CONSTEXPR into Qw-register w/ optional zero-extend.
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
Binary file not shown.
BIN
View File
Binary file not shown.
Binary file not shown.
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
Binary file not shown.
BIN
View File
Binary file not shown.
Binary file not shown.
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
Binary file not shown.
+71 -55
View File
@@ -5,21 +5,19 @@
#
# - `make`
# - Backtraces
# - Syscall tracing
# - Function tracing
# - Reasonably small
# - Reasonably optimized
# - Reasonably debuggable
ifeq ($(MODE),)
CONFIG_CCFLAGS += \
$(BACKTRACES) \
$(FTRACE) \
-Og
-DSYSDEBUG \
-O2
TARGET_ARCH ?= \
-msse3
endif
# Optimized Mode
@@ -27,25 +25,47 @@ endif
# - `make MODE=opt`
# - Backtraces
# - More optimized
# - Syscall tracing
# - Function tracing
# - Reasonably small
# - No memory corruption detection
# - assert() / CHECK_xx() may leak code into binary for debuggability
# - GCC 8+ hoists check fails into .text.cold, thus minimizing impact
ifeq ($(MODE), opt)
CONFIG_CPPFLAGS += \
-DNDEBUG \
-msse2avx \
-Wa,-msse2avx
CONFIG_CCFLAGS += \
$(BACKTRACES) \
-O3
$(FTRACE) \
-DSYSDEBUG \
-O3 \
-fmerge-all-constants
TARGET_ARCH ?= \
-march=native
endif
# Optimized Linux Mode
#
# - `make MODE=optlinux`
# - Turns on red zone
# - Turns off backtraces
# - Turns off function tracing
# - Turns off support for older cpu models
# - Turns off support for other operating systems
ifeq ($(MODE), optlinux)
CONFIG_CPPFLAGS += \
-DNDEBUG \
-msse2avx \
-Wa,-msse2avx \
-DSUPPORT_VECTOR=1
CONFIG_CCFLAGS += \
-O3 -fmerge-all-constants
DEFAULT_COPTS += \
-mred-zone
TARGET_ARCH ?= \
-march=native
endif
# Release Mode
@@ -61,19 +81,16 @@ endif
# - DCHECK_xx() statements removed
# - No memory corruption detection
# - CHECK_xx() won't leak strings into binary
ifeq ($(MODE), rel)
CONFIG_CPPFLAGS += \
-DNDEBUG
CONFIG_CCFLAGS += \
$(BACKTRACES) \
-O2
TARGET_ARCH ?= \
-msse3
PYFLAGS += \
-O1
endif
# Asan Mode
@@ -86,19 +103,14 @@ endif
# - Backtraces
# - Debuggability
# - Larger binaries
ifeq ($(MODE), asan)
CONFIG_CCFLAGS += \
$(BACKTRACES) \
-O2
CONFIG_COPTS += \
-fsanitize=address
TARGET_ARCH ?= \
-msse3
endif
# Debug Mode
@@ -110,26 +122,22 @@ endif
# - Stack canaries
# - No optimization (TODO)
# - Enormous binaries
ifeq ($(MODE), dbg)
CONFIG_CPPFLAGS += \
-DMODE_DBG
CONFIG_CCFLAGS += \
$(BACKTRACES) \
$(FTRACE) \
-O2
-DSYSDEBUG \
-O2 \
-fno-inline
CONFIG_COPTS += \
-fsanitize=address
-fsanitize=address \
-fsanitize=undefined
TARGET_ARCH ?= \
-msse3
OVERRIDE_CCFLAGS += \
-fno-pie
endif
# Tiny Mode
@@ -143,20 +151,26 @@ endif
# - No backtraces
# - No algorithmics
# - YOLO
ifeq ($(MODE), tiny)
CONFIG_CPPFLAGS += \
-DTINY \
-DNDEBUG \
CONFIG_CPPFLAGS += \
-DTINY \
-DNDEBUG \
-DTRUSTWORTHY
CONFIG_CCFLAGS += \
-Os \
-fno-align-functions \
-fno-align-jumps \
-fno-align-labels \
-fno-align-loops
TARGET_ARCH ?= \
CONFIG_CCFLAGS += \
-Os \
-fno-align-functions \
-fno-align-jumps \
-fno-align-labels \
-fno-align-loops \
-fschedule-insns2 \
-fomit-frame-pointer \
-momit-leaf-frame-pointer \
-foptimize-sibling-calls
TARGET_ARCH ?= \
-msse3
PYFLAGS += \
-O2 \
-B
endif
# Linux-Only Tiny Mode
@@ -171,20 +185,21 @@ endif
# - No portability
# - No algorithmics
# - YOLO
ifeq ($(MODE), tinylinux)
CONFIG_CPPFLAGS += \
-DTINY \
-DNDEBUG \
-DTRUSTWORTHY \
CONFIG_CPPFLAGS += \
-DTINY \
-DNDEBUG \
-DTRUSTWORTHY \
-DSUPPORT_VECTOR=1
CONFIG_CCFLAGS += \
-Os \
-fno-align-functions \
-fno-align-jumps \
-fno-align-labels \
DEFAULT_COPTS += \
-mred-zone
CONFIG_CCFLAGS += \
-Os \
-fno-align-functions \
-fno-align-jumps \
-fno-align-labels \
-fno-align-loops
TARGET_ARCH ?= \
TARGET_ARCH ?= \
-msse3
endif
@@ -201,13 +216,14 @@ endif
# - No backtraces
# - No algorithmics
# - YOLO
ifeq ($(MODE), tinylinuxbsd)
CONFIG_CPPFLAGS += \
-DTINY \
-DNDEBUG \
-DTRUSTWORTHY \
-DSUPPORT_VECTOR=113
DEFAULT_COPTS += \
-mred-zone
CONFIG_CCFLAGS += \
-Os \
-fno-align-functions \
@@ -230,13 +246,14 @@ endif
# - No backtraces
# - No algorithmics
# - YOLO
ifeq ($(MODE), tinysysv)
CONFIG_CPPFLAGS += \
-DTINY \
-DNDEBUG \
-DTRUSTWORTHY \
-DSUPPORT_VECTOR=121
DEFAULT_COPTS += \
-mred-zone
CONFIG_CCFLAGS += \
-Os \
-fno-align-functions \
@@ -259,7 +276,6 @@ endif
# - No backtraces
# - No algorithmics
# - YOLO
ifeq ($(MODE), tinynowin)
CONFIG_CPPFLAGS += \
-DTINY \
@@ -279,7 +295,7 @@ endif
# LLVM Mode
ifeq ($(MODE), llvm)
TARGET_ARCH ?= -msse3
CONFIG_CCFLAGS += $(BACKTRACES) $(FTRACE) -O2
CONFIG_CCFLAGS += $(BACKTRACES) $(FTRACE) -DSYSDEBUG -O2
AS = clang
CC = clang
CXX = clang++
+61 -24
View File
@@ -42,7 +42,6 @@
# ASFLAGS assembler flags (don't use -Wa, frontend prefix)
# TARGET_ARCH microarchitecture flags (e.g. -march=native)
V ?= 1
LC_ALL = C
SOURCE_DATE_EPOCH = 0
@@ -51,15 +50,43 @@ ARFLAGS = rcsD
ZFLAGS ?=
XARGS ?= xargs -P4 -rs8000
DOT ?= dot
GZ ?= gzip
CLANG = clang
FC = gfortran #/opt/cross9f/bin/x86_64-linux-musl-gfortran
TMPDIR = o/tmp
# see build/compile, etc. which run third_party/gcc/unbundle.sh
AR = build/bootstrap/ar.com
CP = build/bootstrap/cp.com
RM = build/bootstrap/rm.com -f
ECHO = build/bootstrap/echo.com
TOUCH = build/bootstrap/touch.com
PKG = build/bootstrap/package.com
MKDEPS = build/bootstrap/mkdeps.com
ZIPOBJ = build/bootstrap/zipobj.com
FIXUPOBJ = build/bootstrap/fixupobj.com
MKDIR = build/bootstrap/mkdir.com -p
COMPILE = build/bootstrap/compile.com -V9 -P4096 $(QUOTA)
COMMA := ,
PWD := $(shell build/bootstrap/pwd.com)
IMAGE_BASE_VIRTUAL ?= 0x400000
IGNORE := $(shell $(ECHO) -2 ♥cosmo)
IGNORE := $(shell $(MKDIR) o/tmp)
ifneq ("$(wildcard o/third_party/gcc/bin/x86_64-pc-linux-gnu-as.exe)","")
AS = o/third_party/gcc/bin/x86_64-pc-linux-gnu-as.exe
CC = o/third_party/gcc/bin/x86_64-pc-linux-gnu-gcc.exe
CXX = o/third_party/gcc/bin/x86_64-pc-linux-gnu-g++.exe
CXXFILT = o/third_party/gcc/bin/x86_64-pc-linux-gnu-c++filt.exe
LD = o/third_party/gcc/bin/x86_64-pc-linux-gnu-ld.bfd.exe
NM = o/third_party/gcc/bin/x86_64-pc-linux-gnu-nm.exe
GCC = o/third_party/gcc/bin/x86_64-pc-linux-gnu-gcc.exe
STRIP = o/third_party/gcc/bin/x86_64-pc-linux-gnu-strip.exe
OBJCOPY = o/third_party/gcc/bin/x86_64-pc-linux-gnu-objcopy.exe
OBJDUMP = o/third_party/gcc/bin/x86_64-pc-linux-gnu-objdump.exe
ADDR2LINE = $(PWD)/o/third_party/gcc/bin/x86_64-pc-linux-gnu-addr2line.exe
else
IGNORE := $(shell build/bootstrap/unbundle.com)
AS = o/third_party/gcc/bin/x86_64-linux-musl-as
CC = o/third_party/gcc/bin/x86_64-linux-musl-gcc
CXX = o/third_party/gcc/bin/x86_64-linux-musl-g++
@@ -70,34 +97,39 @@ GCC = o/third_party/gcc/bin/x86_64-linux-musl-gcc
STRIP = o/third_party/gcc/bin/x86_64-linux-musl-strip
OBJCOPY = o/third_party/gcc/bin/x86_64-linux-musl-objcopy
OBJDUMP = o/third_party/gcc/bin/x86_64-linux-musl-objdump
ADDR2LINE = o/third_party/gcc/bin/x86_64-linux-musl-addr2line
COMMA := ,
PWD := $(shell pwd)
IMAGE_BASE_VIRTUAL ?= 0x400000
HELLO := $(shell build/hello)
TMPDIR := $(shell build/findtmp)
SPAWNER := $(shell build/getcompile) -V$(shell build/getccversion $(CC))
COMPILE = $(SPAWNER) $(QUOTA)
ADDR2LINE = $(PWD)/o/third_party/gcc/bin/x86_64-linux-musl-addr2line
endif
export ADDR2LINE
export LC_ALL
export MKDIR
export MODE
export SOURCE_DATE_EPOCH
export TMPDIR
export V
ifeq ($(LANDLOCKMAKE_VERSION),)
TMPSAFE = $(TMPDIR)/$(subst /,_,$@).tmp
else
TMPSAFE = $(TMPDIR)/
endif
FTRACE = \
-pg
BACKTRACES = \
-fno-schedule-insns2 \
-fno-omit-frame-pointer \
-fno-optimize-sibling-calls \
-mno-omit-leaf-frame-pointer
SANITIZER = \
-fsanitize=address
NO_MAGIC = \
-mno-fentry \
-fno-stack-protector \
-fno-sanitize=all \
-fwrapv
-fwrapv \
-fno-sanitize=all
OLD_CODE = \
-fno-strict-aliasing \
@@ -111,7 +143,7 @@ TRADITIONAL = \
DEFAULT_CCFLAGS = \
-Wall \
-Werror \
-fdebug-prefix-map="$(PWD)"= \
-fdebug-prefix-map='$(PWD)'= \
-frecord-gcc-switches
DEFAULT_OFLAGS = \
@@ -128,9 +160,8 @@ DEFAULT_COPTS = \
-fno-gnu-unique \
-fstrict-aliasing \
-fstrict-overflow \
-fno-omit-frame-pointer \
-fno-semantic-interposition \
-mno-omit-leaf-frame-pointer
-mno-tls-direct-seg-refs
MATHEMATICAL = \
-O3 \
@@ -141,19 +172,19 @@ DEFAULT_CPPFLAGS = \
-DMODE='"$(MODE)"' \
-DIMAGE_BASE_VIRTUAL=$(IMAGE_BASE_VIRTUAL) \
-nostdinc \
-iquote.
-iquote .
DEFAULT_CFLAGS = \
-std=gnu2x
DEFAULT_CXXFLAGS = \
-std=gnu++11 \
-fno-rtti \
-fno-exceptions \
-fuse-cxa-atexit \
-fno-threadsafe-statics \
-Wno-int-in-bool-context \
-Wno-narrowing
-Wno-narrowing \
-Wno-literal-suffix
DEFAULT_ASFLAGS = \
-W \
@@ -164,19 +195,22 @@ DEFAULT_ASFLAGS = \
DEFAULT_LDFLAGS = \
-static \
-nostdlib \
-m elf_x86_64 \
-melf_x86_64 \
--gc-sections \
--build-id=none \
--no-dynamic-linker \
-z max-page-size=0x1000
-zmax-page-size=0x1000 #--cref [email protected]
ZIPOBJ_FLAGS = \
-b$(IMAGE_BASE_VIRTUAL)
PYFLAGS = \
-b$(IMAGE_BASE_VIRTUAL)
ASONLYFLAGS = \
-c \
-g \
--debug-prefix-map="$(PWD)"=
--debug-prefix-map='$(PWD)'=
DEFAULT_LDLIBS =
@@ -297,6 +331,9 @@ OBJECTIFY.greg.c = \
-fno-instrument-functions \
-fno-optimize-sibling-calls \
-fno-sanitize=all \
-ffreestanding \
-mno-fentry \
-fwrapv \
-c
OBJECTIFY.ansi.c = $(CC) $(OBJECTIFY.c.flags) -ansi -Wextra -Werror -pedantic-errors -c
-17
View File
@@ -1,17 +0,0 @@
#!/bin/sh
#-*-mode:sh;indent-tabs-mode:nil;tab-width:2;coding:utf-8-*-┐
#───vi: set net ft=sh ts=2 sts=2 fenc=utf-8 :vi─────────────┘
#
# OVERVIEW
#
# Temporary Directory Discovery
#
# DESCRIPTION
#
# We call this script once per build to ideally find a folder that's
# backed by an in-memory file system. We then export it to the TMPDIR
# environment variable. Many programs use it under the hood, e.g. gcc,
# so it grants many small performance improvements.
mkdir -p o/tmp
echo o/tmp
+5 -1
View File
@@ -12,5 +12,9 @@
tail = $(wordlist 2,$(words $1),$1)
reverse = $(if $1,$(call reverse,$(call tail,$1)) $(firstword $1))
uniq = $(if $1,$(firstword $1) $(call uniq,$(filter-out $(firstword $1),$1)))
uniqr = $(if $1,$(call uniqr,$(filter-out $(firstword $1),$1)) $(firstword $1))
# polyfill uniq native (landlock make 1.4)
ifneq ($(call uniq,c b c a),c b a)
uniq = $(if $1,$(firstword $1) $(call uniq,$(filter-out $(firstword $1),$1)))
endif
-39
View File
@@ -1,39 +0,0 @@
#!/bin/sh
#-*-mode:sh;indent-tabs-mode:nil;tab-width:2;coding:utf-8-*-┐
#───vi: set net ft=sh ts=2 sts=2 fenc=utf-8 :vi─────────────┘
#
# OVERVIEW
#
# Compiler Version Discovery
#
# DESCRIPTION
#
# Cosmopolitan itself may be built using either GCC and Clang, and our
# build irons out many of the differences between the two. This script
# is used by build/definitions.mk alongside build/getccname to support
# the different versions folks use.
#
# Our aim is to support GCC 4.2.1+ since that's the last GPLv2 version
# with any sort of industry consensus. Please note, Cosmopolitan never
# links GCC runtimes when using later versions, so some concerns might
# not apply.
if [ ! -d o/third_party/gcc ]; then
third_party/gcc/unbundle.sh
fi
set -e
MAJOR_VERSION=$(
$1 --version |
head -n1 |
sed -n '
s!^[^(]*([^)]*) \([[:digit:]][[:digit:]]*\).*!\1!p
s!^.*clang.*version \([[:digit:]][[:digit:]]*\).*!\1!p
')
if [ -z "$MAJOR_VERSION" ]; then
echo 6
else
printf '%s\n' "$MAJOR_VERSION"
fi
-12
View File
@@ -1,12 +0,0 @@
#!/bin/sh
#-*-mode:sh;indent-tabs-mode:nil;tab-width:2;coding:utf-8-*-┐
#───vi: set net ft=sh ts=2 sts=2 fenc=utf-8 :vi─────────────┘
if ! [ o/build/bootstrap/compile.com -nt build/bootstrap/compile.com ]; then
mkdir -p o/build/bootstrap/
cp -f build/bootstrap/compile.com o/build/bootstrap/compile.$$.com
o/build/bootstrap/compile.$$.com -n
mv -f o/build/bootstrap/compile.$$.com o/build/bootstrap/compile.com
fi
echo o/build/bootstrap/compile.com
-20
View File
@@ -1,20 +0,0 @@
#!/bin/sh
#-*-mode:sh;indent-tabs-mode:nil;tab-width:2;coding:utf-8-*-┐
#───vi: set net ft=sh ts=2 sts=2 fenc=utf-8 :vi─────────────┘
#
# SYNOPSIS
#
# HELLO BUILD
#
# OVERVIEW
#
# We generate a line at the start of each Makefile run, because if we
# use `make V=0` mode then the terminal logging trick we use in
# tool/build/compile.c will delete the previous line, and we'd rather
# have that line not be the bash prompt that ran make.
#
# This script is also useful for giving us an indicator each time the
# build restarts itself from scratch, which can happen in cases like
# when dependencies need to be regenerated by tool/build/mkdeps.c
echo ♥cosmo >&2
+5 -1
View File
@@ -43,12 +43,16 @@
# '(progn
# (add-hook 'c-mode-common-hook 'jart-c-mode-common-hook)))
# ctags doesn't understand atomics, e.g.
# extern char **environ;
set -- --regex-c='/_Atomic(\([^)]*\))/\1/b' "$@"
# ctags doesn't understand variable prototypes, e.g.
# extern char **environ;
set -- --regex-c='/^\(\(hidden\|extern\|const\) \)*[_[:alpha:]][_[:alnum:]]*[ *][ *]*\([_[:alpha:]][_[:alnum:]]*[ *][ *]*\)*\([_[:alpha:]][_$[:alnum:]]*\)/\4/b' "$@"
# ctags doesn't understand function prototypes, e.g.
# bool isheap(void *p) nothrow nocallback;
# bool isheap(void *p) dontthrow nocallback;
set -- --regex-c='/^[_[:alpha:]][_[:alnum:]]*[ *][ *]*\([_[:alpha:]][_[:alnum:]]*[ *][ *]*\)*\([_[:alpha:]][_$[:alnum:]]*\)(.*/\2/b' "$@"
# ctags doesn't understand function pointers, e.g.
+2 -1
View File
@@ -23,8 +23,9 @@
# - tool/build/runitd.c
.PRECIOUS: o/$(MODE)/%.com.ok
o/$(MODE)/%.com.ok: .PLEDGE = stdio rpath wpath cpath proc fattr inet
o/$(MODE)/%.com.ok: \
o/$(MODE)/tool/build/runit.com \
o/$(MODE)/tool/build/runitd.com \
o/$(MODE)/%.com
@$(COMPILE) -ATEST -tT$@ $^ $(HOSTS)
@$(COMPILE) -wATEST -tT$@ $^ $(HOSTS)
+163 -37
View File
@@ -16,70 +16,196 @@
MAKEFLAGS += --no-builtin-rules
o/%.a: ; @$(COMPILE) -AARCHIVE -T$@ $(AR) $(ARFLAGS) $@ $^
o/%.o: %.s ; @$(COMPILE) -AOBJECTIFY.s $(OBJECTIFY.s) $(OUTPUT_OPTION) $<
o/%.o: o/%.s ; @$(COMPILE) -AOBJECTIFY.s $(OBJECTIFY.s) $(OUTPUT_OPTION) $<
o/%.s: %.S ; @$(COMPILE) -APREPROCESS $(PREPROCESS) $(OUTPUT_OPTION) $<
o/%.s: o/%.S ; @$(COMPILE) -APREPROCESS $(PREPROCESS) $(OUTPUT_OPTION) $<
o/%.i: %.S ; @$(COMPILE) -APREPROCESS $(PREPROCESS) $(OUTPUT_OPTION) $<
o/%.o: %.S ; @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) $<
o/%.o: o/%.S ; @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) $<
o/%.s: %.i ; @$(COMPILE) -ACOMPILE.i $(COMPILE.i) $(OUTPUT_OPTION) $<
o/%.s: o/%.i ; @$(COMPILE) -ACOMPILE.i $(COMPILE.i) $(OUTPUT_OPTION) $<
o/%.o: %.cc ; @$(COMPILE) -AOBJECTIFY.cxx $(OBJECTIFY.cxx) $(OUTPUT_OPTION) $<
o/%.o: o/%.cc ; @$(COMPILE) -AOBJECTIFY.cxx $(OBJECTIFY.cxx) $(OUTPUT_OPTION) $<
o/%.lds: %.lds ; @$(COMPILE) -APREPROCESS $(PREPROCESS.lds) $(OUTPUT_OPTION) $<
o/%.inc: %.h ; @$(COMPILE) -APREPROCESS $(PREPROCESS) $(OUTPUT_OPTION) -D__ASSEMBLER__ -P $<
o/%.pkg: ; @$(COMPILE) -APACKAGE -T$@ $(PKG) $(OUTPUT_OPTION) $(addprefix -d,$(filter %.pkg,$^)) $(filter %.o,$^)
o/%.h.ok: %.h ; @$(COMPILE) -ACHECK.h $(COMPILE.c) -x c -g0 -o $@ $<
o/%.h.okk: %.h ; @$(COMPILE) -ACHECK.h $(COMPILE.cxx) -x c++ -g0 -o $@ $<
o/%.greg.o: %.greg.c ; @$(COMPILE) -AOBJECTIFY.greg $(OBJECTIFY.greg.c) $(OUTPUT_OPTION) $<
o/%.zip.o: o/% ; @$(COMPILE) -AZIPOBJ $(ZIPOBJ) $(ZIPOBJ_FLAGS) $(OUTPUT_OPTION) $<
o/$(MODE)/%.a: ; @$(COMPILE) -AARCHIVE -T$@ $(AR) $(ARFLAGS) $@ $^
o/$(MODE)/%: o/$(MODE)/%.dbg ; @$(COMPILE) -AOBJCOPY -T$@ $(OBJCOPY) -S -O binary $< $@
o/$(MODE)/%.o: %.s ; @$(COMPILE) -AOBJECTIFY.s $(OBJECTIFY.s) $(OUTPUT_OPTION) $<
o/$(MODE)/%.o: o/$(MODE)/%.s ; @$(COMPILE) -AOBJECTIFY.s $(OBJECTIFY.s) $(OUTPUT_OPTION) $<
o/$(MODE)/%.s: %.S ; @$(COMPILE) -APREPROCESS $(PREPROCESS) $(OUTPUT_OPTION) $<
o/$(MODE)/%.s: o/$(MODE)/%.S ; @$(COMPILE) -APREPROCESS $(PREPROCESS) $(OUTPUT_OPTION) $<
o/$(MODE)/%.o: %.c ; @$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.o: %.f ; @$(COMPILE) -AOBJECTIFY.f $(OBJECTIFY.f) $(OUTPUT_OPTION) $<
o/$(MODE)/%.o: %.F ; @$(COMPILE) -AOBJECTIFY.F $(OBJECTIFY.F) $(OUTPUT_OPTION) $<
o/$(MODE)/%.o: o/$(MODE)/%.c ; @$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.ss: %.c ; @$(COMPILE) -ACOMPILE.c $(COMPILE.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.ss: o/$(MODE)/%.c ; @$(COMPILE) -AOBJECTIFY.s $(COMPILE.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.i: %.S ; @$(COMPILE) -APREPROCESS $(PREPROCESS) $(OUTPUT_OPTION) $<
o/$(MODE)/%.i: %.c ; @$(COMPILE) -APREPROCESS $(PREPROCESS) $(OUTPUT_OPTION) $<
o/$(MODE)/%.i: %.cc ; @$(COMPILE) -APREPROCESS $(PREPROCESS) $(OUTPUT_OPTION) $<
o/$(MODE)/%.i: o/$(MODE)/%.c ; @$(COMPILE) -APREPROCESS $(PREPROCESS) $(OUTPUT_OPTION) $<
o/$(MODE)/%.h: %.c ; @$(COMPILE) -AAMALGAMATE $(PREPROCESS) $(OUTPUT_OPTION) -fdirectives-only -P $<
o/$(MODE)/%.h: o/$(MODE)/%.c ; @$(COMPILE) -AAMALGAMATE $(PREPROCESS) $(OUTPUT_OPTION) -fdirectives-only -P $<
o/$(MODE)/%.o: %.S ; @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) $<
o/$(MODE)/%.o: o/$(MODE)/%.S ; @$(COMPILE) -AOBJECTIFY.S $(OBJECTIFY.S) $(OUTPUT_OPTION) $<
o/$(MODE)/%.s: %.i ; @$(COMPILE) -ACOMPILE.i $(COMPILE.i) $(OUTPUT_OPTION) $<
o/$(MODE)/%.s: o/$(MODE)/%.i ; @$(COMPILE) -ACOMPILE.i $(COMPILE.i) $(OUTPUT_OPTION) $<
o/$(MODE)/%.o: %.cc ; @$(COMPILE) -AOBJECTIFY.cxx $(OBJECTIFY.cxx) $(OUTPUT_OPTION) $<
o/$(MODE)/%.o: o/$(MODE)/%.cc ; @$(COMPILE) -AOBJECTIFY.cxx $(OBJECTIFY.cxx) $(OUTPUT_OPTION) $<
o/$(MODE)/%.lds: %.lds ; @$(COMPILE) -APREPROCESS $(PREPROCESS.lds) $(OUTPUT_OPTION) $<
o/$(MODE)/%.h.ok: %.h ; @$(COMPILE) -ACHECK.h $(COMPILE.c) -x c -g0 -o $@ $<
o/$(MODE)/%.h.okk: %.h ; @$(COMPILE) -ACHECK.h $(COMPILE.cxx) -x c++ -g0 -o $@ $<
o/$(MODE)/%.o: %.greg.c ; @$(COMPILE) -AOBJECTIFY.greg $(OBJECTIFY.greg.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.greg.o: %.greg.c ; @$(COMPILE) -AOBJECTIFY.greg $(OBJECTIFY.greg.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.ansi.o: %.ansi.c ; @$(COMPILE) -AOBJECTIFY.ansi $(OBJECTIFY.ansi.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.ansi.o: %.c ; @$(COMPILE) -AOBJECTIFY.ansi $(OBJECTIFY.ansi.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.c99.o: %.c99.c ; @$(COMPILE) -AOBJECTIFY.c99 $(OBJECTIFY.c99.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.c11.o: %.c11.c ; @$(COMPILE) -AOBJECTIFY.c11 $(OBJECTIFY.c11.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.c2x.o: %.c2x.c ; @$(COMPILE) -AOBJECTIFY.c2x $(OBJECTIFY.c2x.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.initabi.o: %.initabi.c ; @$(COMPILE) -AOBJECTIFY.init $(OBJECTIFY.initabi.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.ncabi.o: %.ncabi.c ; @$(COMPILE) -AOBJECTIFY.nc $(OBJECTIFY.ncabi.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.real.o: %.c ; @$(COMPILE) -AOBJECTIFY.real $(OBJECTIFY.real.c) $(OUTPUT_OPTION) $<
o/$(MODE)/%.runs: o/$(MODE)/% ; @$(COMPILE) -ACHECK -tT$@ $< $(TESTARGS)
o/$(MODE)/%.pkg: ; @$(COMPILE) -APACKAGE -T$@ $(PKG) $(OUTPUT_OPTION) $(addprefix -d,$(filter %.pkg,$^)) $(filter %.o,$^)
o/$(MODE)/%.zip.o: % ; @$(COMPILE) -AZIPOBJ $(ZIPOBJ) $(ZIPOBJ_FLAGS) $(OUTPUT_OPTION) $<
o/$(MODE)/%-gcc.asm: %.c ; @$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.c) -S -g0 $(OUTPUT_OPTION) $<
o/$(MODE)/%-clang.asm: %.c ; @$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.c) -S -g0 $(OUTPUT_OPTION) $< || echo / need $(CLANG) >$@
o/$(MODE)/%-gcc.asm: %.cc ; @$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.cxx) -S -g0 $(OUTPUT_OPTION) $<
o/$(MODE)/%-clang.asm: %.c ; @$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.c) -S -g0 $(OUTPUT_OPTION) $<
o/$(MODE)/%-clang.asm: %.cc ; @$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.cxx) -S -g0 $(OUTPUT_OPTION) $<
o/$(MODE)/%-clang.asm: CC = $(CLANG)
o/%.o: %.cc
@$(COMPILE) -AOBJECTIFY.cxx $(OBJECTIFY.cxx) $(OUTPUT_OPTION) $<
@$(COMPILE) -AFIXUPOBJ -wT$@ $(FIXUPOBJ) $@
o/%.o: o/%.cc
@$(COMPILE) -AOBJECTIFY.cxx $(OBJECTIFY.cxx) $(OUTPUT_OPTION) $<
@$(COMPILE) -AFIXUPOBJ -wT$@ $(FIXUPOBJ) $@
o/$(MODE)/%.o: %.c
@$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.c) $(OUTPUT_OPTION) $<
@$(COMPILE) -AFIXUPOBJ -wT$@ $(FIXUPOBJ) $@
o/$(MODE)/%.o: o/$(MODE)/%.c
@$(COMPILE) -AOBJECTIFY.c $(OBJECTIFY.c) $(OUTPUT_OPTION) $<
@$(COMPILE) -AFIXUPOBJ -wT$@ $(FIXUPOBJ) $@
o/$(MODE)/%.o: o/$(MODE)/%.cc
@$(COMPILE) -AOBJECTIFY.cxx $(OBJECTIFY.cxx) $(OUTPUT_OPTION) $<
@$(COMPILE) -AFIXUPOBJ -wT$@ $(FIXUPOBJ) $@
o/%.a:
$(file >$(TMPDIR)/$(subst /,_,$@),$^)
@$(COMPILE) -AARCHIVE -wT$@ $(AR) $(ARFLAGS) $@ @$(TMPDIR)/$(subst /,_,$@)
o/%.pkg:
$(file >$(TMPSAFE).args,$(filter %.o,$^))
@$(COMPILE) -APACKAGE -wT$@ $(PKG) $(OUTPUT_OPTION) $(addprefix -d,$(filter %.pkg,$^)) @$(TMPSAFE).args
o/$(MODE)/%.pkg:
$(file >$(TMPSAFE).args,$(filter %.o,$^))
@$(COMPILE) -APACKAGE -wT$@ $(PKG) $(OUTPUT_OPTION) $(addprefix -d,$(filter %.pkg,$^)) @$(TMPSAFE).args
o/$(MODE)/%.o: %.py o/$(MODE)/third_party/python/pyobj.com
@$(COMPILE) -wAPYOBJ o/$(MODE)/third_party/python/pyobj.com $(PYFLAGS) -o $@ $<
o/$(MODE)/%.pyc: %.py o/$(MODE)/third_party/python/pycomp.com
@$(COMPILE) -wAPYCOMP o/$(MODE)/third_party/python/pycomp.com $(PYCFLAGS) -o $@ $<
o/$(MODE)/%.lua: %.lua o/$(MODE)/third_party/lua/luac.com
@$(COMPILE) -ALUAC -T$@ o/$(MODE)/third_party/lua/luac.com -s -o $@ $<
@$(COMPILE) -wALUAC o/$(MODE)/third_party/lua/luac.com -s -o $@ $<
o/$(MODE)/%-clang.asm: CC = $(CLANG)
o/$(MODE)/%.lua.runs: %.lua o/$(MODE)/tool/net/redbean.com
@$(COMPILE) -wALUA -tT$@ o/$(MODE)/tool/net/redbean.com $(LUAFLAGS) -i $<
o/$(MODE)/%: o/$(MODE)/%.com o/$(MODE)/tool/build/cp.com o/$(MODE)/tool/build/assimilate.com
@$(COMPILE) -wACP -T$@ o/$(MODE)/tool/build/cp.com $< $@
@$(COMPILE) -wAASSIMILATE -T$@ o/$(MODE)/tool/build/assimilate.com $@
################################################################################
# LOCAL UNIT TESTS
#
# We always run unit tests as part of the normal `make` invocation. you
# may override the $(TESTARGS) variable to do things such as enable the
# benchmarking feature. For example:
#
# TESTARGS = -b
#
# May be specified in your ~/.cosmo.mk file. You can also use this to
# enable things like function tracing. For example:
#
# TESTARGS = ----ftrace
# .PLEDGE += prot_exec
#
# You could then run a command like:
#
# make -j8 o//test/libc/calls/openbsd_test.com.runs
#
# You need PROT_EXEC permission since ftrace morphs the binary. It's
# also worth mentioning that the pledge.com command can simulate what
# Landlock Make does:
#
# o//tool/build/pledge.com \
# -v. -p 'stdio rpath wpath cpath tty prot_exec' \
# o//test/libc/calls/openbsd_test.com \
# ----ftrace
#
# This is useful in the event a test binary should run by itself, but
# fails to run beneath Landlock Make. It's also useful sometimes to
# override the verbosity when running tests:
#
# make V=5 TESTARGS=-b o//test/libc/calls/openbsd_test.com.runs
#
# This way, if for some reason a test should fail but calls exit(0),
# then the stdout/stderr output, which would normally be suppressed,
# will actually be displayed.
o/$(MODE)/%.runs: o/$(MODE)/%
@$(COMPILE) -ACHECK -wtT$@ $< $(TESTARGS)
################################################################################
# ELF ZIP FILES
#
# zipobj.com lets us do fast incremental linking of compressed data.
# it's nice because if we link a hundred binaries that use the time zone
# database, then that database only needs to be DEFLATE'd once.
o/%.zip.o: o/%
@$(COMPILE) -wAZIPOBJ $(ZIPOBJ) $(ZIPOBJ_FLAGS) $(OUTPUT_OPTION) $<
o/$(MODE)/%.zip.o: %
@$(COMPILE) -wAZIPOBJ $(ZIPOBJ) $(ZIPOBJ_FLAGS) $(OUTPUT_OPTION) $<
o/$(MODE)/%.zip.o: %
@$(COMPILE) -wAZIPOBJ $(ZIPOBJ) $(ZIPOBJ_FLAGS) $(OUTPUT_OPTION) $<
# an issue with sandboxing arises when creating directory entries in the
# zip file. we need the trailing slash (e.g. o//foo/.zip.o) but Landlock
# Make avoids sandboxing directory names that have a trailing slash (so
# they can be used to watch for deleted files, without creating overly
# broad unveiling). such rules need to be written more explicitly.
o/$(MODE)%/.zip.o: %
@$(COMPILE) -wAZIPOBJ $(ZIPOBJ) $(ZIPOBJ_FLAGS) $(OUTPUT_OPTION) $<
################################################################################
# STRICT HEADER CHECKING
#
# these rules are unsandboxed since they're already a sandboxing test,
# and it would be too costly in terms of make latency to have every
# header file depend on $(HDRS) and $(INCS).
o/%.h.ok: .UNSANDBOXED = 1
o/%.h.ok: %.h
@$(COMPILE) -ACHECK.h $(COMPILE.c) -xc -g0 -o $@ $<
o/$(MODE)/%.h.ok: .UNSANDBOXED = 1
o/$(MODE)/%.h.ok: %.h
@$(COMPILE) -ACHECK.h $(COMPILE.c) -xc -g0 -o $@ $<
o/$(MODE)/%.hh.ok: .UNSANDBOXED = 1
o/$(MODE)/%.hh.ok: %.hh
@$(COMPILE) -ACHECK.h $(COMPILE.cxx) -xc++ -g0 -o $@ $<
o/%.okk: .UNSANDBOXED = 1
o/%.okk: %
@$(COMPILE) -ACHECK.h $(COMPILE.cxx) -xc++ -g0 -o $@ $<
o/$(MODE)/%.okk: .UNSANDBOXED = 1
o/$(MODE)/%.okk: %
@$(COMPILE) -ACHECK.h $(COMPILE.cxx) -xc++ -g0 -o $@ $<
################################################################################
# EXECUTABLE HELPERS
MAKE_OBJCOPY = \
$(COMPILE) -AOBJCOPY -T$@ \
$(OBJCOPY) -S -O binary $< $@
MAKE_SYMTAB_CREATE = \
$(COMPILE) -wASYMTAB \
o/$(MODE)/tool/build/symtab.com \
-o $(TMPSAFE)/.symtab \
$<
MAKE_SYMTAB_ZIP = \
$(COMPILE) -AZIP -T$@ \
o/$(MODE)/third_party/zip/zip.com \
-b$(TMPDIR) \
-9qj \
$@ \
$(TMPSAFE)/.symtab
+18 -10
View File
@@ -27,29 +27,37 @@ EOF
exit 1
fi
build/sanitycheck2
if [ $? -ne 123 ]; then
if [ ! -f /proc/sys/fs/binfmt_misc/status ]; then
exit 0
fi
if ! build/bootstrap/echo.com -n; then
cat <<'EOF' >&2
ERROR
Thompson Shell Backwards Compatibility Issue Detected
APE Execution error.
DETAILS
Actually Portable Executable assumes stock Linux configuration.
Normal behavior is non-ELF files with x bit are run by /bin/sh.
Linux lets people globally define arbitrary magic interpreters.
Your computer couldve been tuned to run MZ scripts inside WINE.
So if you use binfmt_misc you need to explicitly register this.
Your system has probably been configured to use binfmt_misc. You need
to run the command below to install /usr/bin/ape and register it with
binfmt_misc. See ape/loader.c for source code, and technical details.
WORKAROUND
sudo sh -c "echo ':APE:M::MZqFpD::/bin/sh:' >/proc/sys/fs/binfmt_misc/register"
ape/apeinstall.sh
NOTES
If it still doesn't work, possibly due to an overly aggressive WINE
or WSL registration, or possibly due to the ordering of definitions,
then one troubleshooting step is to just unregister everything using
sudo sh -c 'echo -1 >/proc/sys/fs/binfmt_misc/status' and try again.
SEE ALSO
https://justine.storage.googleapis.com/ape.html
https://justine.lol/ape.html
EOF
kill $1
-9
View File
@@ -1,9 +0,0 @@
MZqFpD=123
exit $MZqFpD
+2 -2
View File
@@ -40,11 +40,11 @@ $(DSP_BMP_A).pkg: \
$(foreach x,$(DSP_BMP_A_DIRECTDEPS),$($(x)_A).pkg)
o/$(MODE)/dsp/bmp/float2short.o \
o/$(MODE)/dsp/bmp/scalevolume.o: \
o/$(MODE)/dsp/bmp/scalevolume.o: private \
OVERRIDE_CFLAGS += \
$(MATHEMATICAL)
o/tiny/dsp/bmp/scalevolume.o: \
o/tiny/dsp/bmp/scalevolume.o: private \
OVERRIDE_CFLAGS += \
-Os
+46
View File
@@ -0,0 +1,46 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/core/core.h"
/**
* Compresses 16-bit signed linear audio sample with a-Law.
*
* @param x is clamped to 16-bits of which the top 14 are considered
* @return coded number in range [0..255]
* @see ITU G.711
*/
int alaw(int x) {
int a, b, e, i;
if ((a = x) < 0) a = ~a;
a >>= 4;
if (a > 15) {
if ((i = a >> 5)) {
b = (__builtin_clz(i) ^ 31) + 1;
a >>= b;
a -= 16;
a += (b + 1) << 4;
} else {
e = 1;
a -= 16;
a += 16;
}
}
if (x >= 0) a |= 128;
return a ^ 85;
}
+1 -1
View File
@@ -22,7 +22,7 @@
void *byte2double(long n, const void *p, double weight, double bias) {
long i;
double f, *dst;
unsigned char *src;
const unsigned char *src;
if ((dst = valloc(n * sizeof(double)))) {
for (src = p, i = 0; i < n; ++i) {
f = src[i];
+3 -7
View File
@@ -3,23 +3,19 @@
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
/**
* @fileoverview Cosmopolitan Digital Signal Processing.
*/
int alaw(int);
int unalaw(int);
int mulaw(int);
int unmulaw(int);
void *double2byte(long, const void *, double, double) vallocesque;
void *byte2double(long, const void *, double, double) vallocesque;
void *dct(float[8][8], float, float, float, float, float);
void *dctjpeg(float[8][8]);
double det3(const double[3][3]) nosideeffect;
void *inv3(double[restrict 3][3], const double[restrict 3][3], double);
void *matmul3(double[restrict 3][3], const double[3][3], const double[3][3]);
void *vmatmul3(double[restrict 3], const double[3], const double[3][3]);
void *matvmul3(double[restrict 3], const double[3][3], const double[3]);
double rgb2stdtv(double) pureconst;
double rgb2lintv(double) pureconst;
double rgb2stdpc(double, double) pureconst;
+3 -3
View File
@@ -47,15 +47,15 @@ o/$(MODE)/dsp/core/c1331.o \
o/$(MODE)/dsp/core/magikarp.o \
o/$(MODE)/dsp/core/c93654369.o \
o/$(MODE)/dsp/core/float2short.o \
o/$(MODE)/dsp/core/scalevolume.o: \
o/$(MODE)/dsp/core/scalevolume.o: private \
OVERRIDE_CFLAGS += \
$(MATHEMATICAL)
o/tiny/dsp/core/scalevolume.o: \
o/tiny/dsp/core/scalevolume.o: private \
OVERRIDE_CFLAGS += \
-Os
o/$(MODE)/dsp/core/det3.o: \
o/$(MODE)/dsp/core/det3.o: private \
OVERRIDE_CFLAGS += \
-ffast-math
+1 -1
View File
@@ -21,7 +21,7 @@
#define DCT(A, B, C, D, E, F, G, H, T, C0, C1, C2, C3, C4) \
do { \
T z1, z2, z3, z4, z5, z11, z13; \
T t0, t1, t2, t3, t4, t5, t6, t7, t8, t10, t11, t12, t13; \
T t0, t1, t2, t3, t4, t5, t6, t7, t10, t11, t12, t13; \
t0 = A + H; \
t7 = A - H; \
t1 = B + G; \
-1
View File
@@ -30,7 +30,6 @@
*/
void float2short(size_t n, short pcm16[n][8], const float binary32[n][8]) {
size_t i, j;
float f[8], w[8];
for (i = 0; i < n; ++i) {
for (j = 0; j < 8; ++j) {
pcm16[i][j] =
-63
View File
@@ -1,63 +0,0 @@
/*-*- mode:unix-assembly; indent-tabs-mode:t; tab-width:8; coding:utf-8 -*-│
│vi: set et ft=asm ts=8 tw=8 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2020 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
#define BIAS 0x84
// Encodes audio sample with µ-Law.
//
// This is both the highest quality and most widely supported
// telephony codec, whose use was phased out in the 2000's in
// favor of cost-saving GSM audio compression that was so bad
// consumers were willing to pay more cash, for the privilege
// of saving telcos even more money w/ text messaging. Mu Law
// reduces PCM data to half its original size, by diminishing
// audio bands not vocalized by human voice.
//
// @param %edi is pcm sample
// @return %eax is uint8_t encoded sample
mulaw: .leafprologue
.profilable
mov $BIAS,%eax
xor %edx,%edx
test %edi,%edi
js 1f
lea (%edi,%eax),%eax
mov $0xFF,%dl
jmp 2f
1: sub %edi,%eax
mov $0x7F,%dl
2: mov %eax,%esi
or $0xFF,%sil
bsr %esi,%esi
sub $7,%esi
cmp $8,%esi
jge 4f
lea 3(%rdx),%ecx
sar %cl,%eax
and $0xF,%eax
shl $4,%esi
or %esi,%eax
xor %edx,%eax
3: .leafepilogue
4: xor $0x7F,%dl
mov %edx,%eax
jmp 3b
.endfn mulaw,globl
.source __FILE__
+40
View File
@@ -0,0 +1,40 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/core/core.h"
/**
* Compresses 16-bit signed linear audio sample with μ-Law.
*
* @param x is clamped to 16-bits of which the top 14 are considered
* @return coded number in range [0..255]
* @see ITU G.711
*/
int mulaw(int x) {
int b, i, a, s, l, h;
a = x < 0 ? (~x >> 2) + 33 : (x >> 2) + 33;
if (a > 8191) a = 8191;
i = a >> 6;
s = i ? (__builtin_clz(i) ^ 31) + 2 : 1;
h = 8 - s;
l = (a >> s) & 15;
l = 15 - l;
b = (h << 4) | l;
if (x >= 0) b |= 128;
return b;
}
-1
View File
@@ -37,4 +37,3 @@ sad16x8n:
jnz 0b
1: .leafepilogue
.endfn sad16x8n,globl,hidden
.source __FILE__
+2 -2
View File
@@ -17,8 +17,8 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/core/core.h"
#include "libc/bits/bits.h"
#include "libc/bits/safemacros.internal.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/limits.h"
/**
+38
View File
@@ -0,0 +1,38 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/core/core.h"
/**
* Expands a-Law coded to audio sample.
*
* @param x is coded number masked to range [0..255]
* @return 16-bit signed linear audio sample
* @note generate lut to save 2 cycles
* @see ITU G.711
*/
int unalaw(int x) {
int e, i, m;
i = (x ^ 85) & 127;
e = i >> 4;
m = i & 15;
if (e > 0) m += 16;
m = (m << 4) + 8;
if (e > 1) m = m << (e - 1);
return x & 128 ? m : -m;
}
+37
View File
@@ -0,0 +1,37 @@
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
│vi: set net ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi│
╞══════════════════════════════════════════════════════════════════════════════╡
│ Copyright 2021 Justine Alexandra Roberts Tunney │
│ │
│ Permission to use, copy, modify, and/or distribute this software for │
│ any purpose with or without fee is hereby granted, provided that the │
│ above copyright notice and this permission notice appear in all copies. │
│ │
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/core/core.h"
#include "libc/intrin/likely.h"
/**
* Expands μ-Law coded to audio sample.
*
* @param x is coded number masked to range [0..255]
* @return 16-bit signed linear audio sample
* @note generate lut to save 2 cycles
* @see ITU G.711
*/
int unmulaw(int x) {
int e, m, k, s;
e = (~x >> 4) & 7;
m = ~x & 15;
k = 4 << (e + 1);
s = (128 << e) + k * m + k / 2 - 4 * 33;
return x & 128 ? s : -s;
}
+5 -5
View File
@@ -37,10 +37,10 @@ This library provides several interfaces to load, demux and decode MPEG video
and audio data. A high-level API combines the demuxer, video & audio decoders
in an easy to use wrapper.
Lower-level APIs for accessing the demuxer, video decoder and audio decoder,
Lower-level APIs for accessing the demuxer, video decoder and audio decoder,
as well as providing different data sources are also available.
Interfaces are written in an object orientet style, meaning you create object
Interfaces are written in an object orientet style, meaning you create object
instances via various different constructor functions (plm_*create()),
do some work on them and later dispose them via plm_*destroy().
@@ -51,11 +51,11 @@ plm_video_* -- the MPEG1 Video ("mpeg1") decoder
plm_audio_* -- the MPEG1 Audio Layer II ("mp2") decoder
This library uses malloc(), realloc() and free() to manage memory. Typically
This library uses malloc(), realloc() and free() to manage memory. Typically
all allocation happens up-front when creating the interface. However, the
default buffer size may be too small for certain inputs. In these cases plmpeg
will realloc() the buffer with a larger size whenever needed. You can configure
the default buffer size by defining PLM_BUFFER_DEFAULT_SIZE *before*
the default buffer size by defining PLM_BUFFER_DEFAULT_SIZE *before*
including this library.
With the high-level interface you have two options to decode video & audio:
@@ -85,7 +85,7 @@ mat4 rec601 = mat4(
gl_FragColor = vec4(y, cb, cr, 1.0) * rec601;
Audio data is decoded into a struct with either one single float array with the
samples for the left and right channel interleaved, or if the
samples for the left and right channel interleaved, or if the
PLM_AUDIO_SEPARATE_CHANNELS is defined *before* including this library, into
two separate float arrays - one for each channel.
+2 -2
View File
@@ -67,7 +67,7 @@ plm_buffer_t *plm_buffer_create_with_file(FILE *fh, int close_when_done) {
plm_buffer_t *plm_buffer_create_with_memory(unsigned char *bytes, size_t length, int free_when_done) {
plm_buffer_t *b;
b = memalign(alignof(plm_buffer_t), sizeof(plm_buffer_t));
b = memalign(_Alignof(plm_buffer_t), sizeof(plm_buffer_t));
memset(b, 0, sizeof(plm_buffer_t));
b->capacity = length;
b->length = length;
@@ -79,7 +79,7 @@ plm_buffer_t *plm_buffer_create_with_memory(unsigned char *bytes, size_t length,
plm_buffer_t * plm_buffer_create_with_capacity(size_t capacity) {
plm_buffer_t *b;
b = memalign(alignof(plm_buffer_t), sizeof(plm_buffer_t));
b = memalign(_Alignof(plm_buffer_t), sizeof(plm_buffer_t));
memset(b, 0, sizeof(plm_buffer_t));
b->capacity = capacity;
b->free_when_done = true;
-1
View File
@@ -17,7 +17,6 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
.source __FILE__
clamp4int256$core:
.leafprologue
+2 -2
View File
@@ -115,8 +115,8 @@ plm_packet_t *plm_demux_decode(plm_demux_t *self) {
do {
code = plm_buffer_next_start_code(self->buffer);
if (
code == PLM_DEMUX_PACKET_VIDEO_1 ||
code == PLM_DEMUX_PACKET_PRIVATE ||
code == PLM_DEMUX_PACKET_VIDEO_1 ||
code == PLM_DEMUX_PACKET_PRIVATE ||
(code >= PLM_DEMUX_PACKET_AUDIO_1 && code <= PLM_DEMUX_PACKET_AUDIO_4)
) {
return plm_demux_decode_packet(self, code);
+5 -5
View File
@@ -273,11 +273,11 @@ typedef plm_audio_t plm_audio_t;
int plm_audio_decode_header(plm_audio_t *self);
void plm_audio_decode_frame(plm_audio_t *self);
const plm_quantizer_spec_t *plm_audio_read_allocation(plm_audio_t *self, int sb, int tab3);
void plm_audio_read_samples(plm_audio_t *self, int ch, int sb, int part);
void plm_audio_read_samples(plm_audio_t *self, int ch, int sb, int part);
void plm_audio_matrix_transform(int s[32][3], int ss, float *d, int dp);
plm_audio_t *plm_audio_create_with_buffer(plm_buffer_t *buffer, int destroy_when_done) {
plm_audio_t *self = (plm_audio_t *)memalign(alignof(plm_audio_t), sizeof(plm_audio_t));
plm_audio_t *self = (plm_audio_t *)memalign(_Alignof(plm_audio_t), sizeof(plm_audio_t));
memset(self, 0, sizeof(plm_audio_t));
self->samples.count = PLM_AUDIO_SAMPLES_PER_FRAME;
@@ -345,7 +345,7 @@ plm_samples_t *plm_audio_decode(plm_audio_t *self) {
self->samples.time = self->time;
self->samples_decoded += PLM_AUDIO_SAMPLES_PER_FRAME;
self->time = (double)self->samples_decoded /
self->time = (double)self->samples_decoded /
(double)PLM_AUDIO_SAMPLE_RATE[self->samplerate_index];
return &self->samples;
@@ -402,7 +402,7 @@ int plm_audio_decode_header(plm_audio_t *self) {
plm_buffer_skip(self->buffer, 16);
}
// Compute frame size, check if we have enough data to decode the whole
// Compute frame size, check if we have enough data to decode the whole
// frame.
int bitrate = PLM_AUDIO_BIT_RATE[self->bitrate_index];
int samplerate = PLM_AUDIO_SAMPLE_RATE[self->samplerate_index];
@@ -561,7 +561,7 @@ void plm_audio_decode_frame(plm_audio_t *self) {
}
#else
for (int j = 0; j < 32; j++) {
self->samples.interleaved[((out_pos + j) << 1) + ch] =
self->samples.interleaved[((out_pos + j) << 1) + ch] =
self->U[j] / 2147418112.0f;
}
#endif
+2 -6
View File
@@ -37,8 +37,7 @@ DSP_MPEG_A_DIRECTDEPS = \
LIBC_STUBS \
LIBC_SYSV \
LIBC_TIME \
LIBC_TINYMATH \
LIBC_UNICODE
LIBC_TINYMATH
DSP_MPEG_A_DEPS := \
$(call uniq,$(foreach x,$(DSP_MPEG_A_DIRECTDEPS),$($(x))))
@@ -51,13 +50,10 @@ $(DSP_MPEG_A).pkg: \
$(DSP_MPEG_A_OBJS) \
$(foreach x,$(DSP_MPEG_A_DIRECTDEPS),$($(x)_A).pkg)
o/$(MODE)/dsp/mpeg/clamp4int256-k8.o: \
o/$(MODE)/dsp/mpeg/clamp4int256-k8.o: private \
OVERRIDE_CFLAGS += \
-Os
#o/$(MODE)/dsp/mpeg/macroblock.o: \
CC = $(CLANG)
DSP_MPEG_LIBS = $(foreach x,$(DSP_MPEG_ARTIFACTS),$($(x)))
DSP_MPEG_SRCS = $(foreach x,$(DSP_MPEG_ARTIFACTS),$($(x)_SRCS))
DSP_MPEG_HDRS = $(foreach x,$(DSP_MPEG_ARTIFACTS),$($(x)_HDRS))
+22 -21
View File
@@ -32,7 +32,7 @@
#include "dsp/mpeg/idct.h"
#include "dsp/mpeg/mpeg.h"
#include "dsp/mpeg/video.h"
#include "libc/bits/initializer.internal.h"
#include "libc/intrin/initializer.internal.h"
#include "libc/fmt/conv.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
@@ -72,7 +72,8 @@ static const float PLM_VIDEO_PIXEL_ASPECT_RATIO[] = {
0.6735, /* 3:4? */
0.7031, /* MPEG-1 / MPEG-2 video encoding divergence? */
0.7615, 0.8055, 0.8437, 0.8935, 0.9157, 0.9815,
1.0255, 1.0695, 1.0950, 1.1575, 1.2051};
1.0255, 1.0695, 1.0950, 1.1575, 1.2051,
};
static const float PLM_VIDEO_PICTURE_RATE[] = {
23.976, /* NTSC-Film */
@@ -591,24 +592,24 @@ void plm_video_decode_sequence_header(plm_video_t *self) {
self->has_sequence_header = true;
LOGF("%s:\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15f;\n"
"\t%-20s = %15f;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;",
"New MPEG Sequence", "width", self->width, "height", self->height,
"framerate", self->framerate, "pixel_aspect_ratio",
self->pixel_aspect_ratio, "mb_size", self->mb_size, "mb_width",
self->mb_width, "mb_height", self->mb_height, "luma_width",
self->luma_width, "luma_height", self->luma_height, "chroma_width",
self->chroma_width, "chroma_height", self->chroma_height);
INFOF("%s:\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15f;\n"
"\t%-20s = %15f;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;\n"
"\t%-20s = %15d;",
"New MPEG Sequence", "width", self->width, "height", self->height,
"framerate", self->framerate, "pixel_aspect_ratio",
self->pixel_aspect_ratio, "mb_size", self->mb_size, "mb_width",
self->mb_width, "mb_height", self->mb_height, "luma_width",
self->luma_width, "luma_height", self->luma_height, "chroma_width",
self->chroma_width, "chroma_height", self->chroma_height);
}
static void plm_video_copy_macroblock(plm_video_t *self, int motion_h,
@@ -1082,7 +1083,7 @@ static plm_frame_t *plm_video_decode_impl(plm_video_t *self) {
plm_frame_t *plm_video_decode(plm_video_t *self) {
long double tsc;
plm_frame_t *res;
LOGF("plm_video_decode");
INFOF("plm_video_decode");
tsc = nowl();
res = plm_video_decode_impl(self);
plmpegdecode_latency_ = lroundl((nowl() - tsc) * 1e6l);
+3 -3
View File
@@ -94,9 +94,9 @@ plm_t *plm_create_with_buffer(plm_buffer_t *buffer, int destroy_when_done) {
self->demux = plm_demux_create(buffer, destroy_when_done);
// In theory we should check plm_demux_get_num_video_streams() and
// In theory we should check plm_demux_get_num_video_streams() and
// plm_demux_get_num_audio_streams() here, but older files typically
// do not specify these correcly. So we just assume we have a video and
// do not specify these correctly. So we just assume we have a video and
// audio stream and create the decoders.
self->video_packet_type = PLM_DEMUX_PACKET_VIDEO_1;
@@ -159,7 +159,7 @@ double plm_get_framerate(plm_t *self) {
int plm_get_num_audio_streams(plm_t *self) {
// Some files do not specify the number of audio streams in the system header.
// If the reported number of streams is 0, we check if we have a samplerate,
// If the reported number of streams is 0, we check if we have a samplerate,
// indicating at least one audio stream.
int num_streams = plm_demux_get_num_audio_streams(self->demux);
return num_streams == 0 && plm_get_samplerate(self) ? 1 : num_streams;
+1 -1
View File
@@ -40,7 +40,7 @@ asm(".include \"libc/disclaimer.inc\"");
* @see YCbCr2RGB() in tool/viz/lib/ycbcr2rgb.c
*/
void plm_frame_to_rgb(plm_frame_t *frame, uint8_t *rgb) {
// Chroma values are the same for each block of 4 pixels, so we proccess
// Chroma values are the same for each block of 4 pixels, so we process
// 2 lines at a time, 2 neighboring pixels each.
int w = frame->y.width, w2 = w >> 1;
int y_index1 = 0, y_index2 = w, y_next_2_lines = w + (w - frame->width);
+1 -1
View File
@@ -101,7 +101,7 @@ struct SamplingSolution *ComputeSamplingSolution(long dn, long sn, double dar,
if (!off) off = (dar - 1) / 2;
f = dar < 1 ? 1 / dar : dar;
s = 3 * f + 4;
fweights = gc(xcalloc(s, sizeof(double)));
fweights = gc(xcalloc(s + /*xxx*/ 2, sizeof(double)));
res = NewSamplingSolution(dn, s);
weights = res->weights;
indices = res->indices;
+1 -1
View File
@@ -51,7 +51,7 @@ $(DSP_SCALE_A).pkg: \
o/$(MODE)/dsp/scale/cdecimate2xuint8x8.o \
o/$(MODE)/dsp/scale/gyarados.o \
o/$(MODE)/dsp/scale/magikarp.o \
o/$(MODE)/dsp/scale/scale.o: \
o/$(MODE)/dsp/scale/scale.o: private \
OVERRIDE_CFLAGS += \
$(MATHEMATICAL)
+3 -2
View File
@@ -17,8 +17,9 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/tty/tty.h"
#include "libc/bits/pushpop.h"
#include "libc/intrin/pushpop.h"
#include "libc/dce.h"
#include "libc/log/internal.h"
#include "libc/log/log.h"
#include "libc/nt/console.h"
#include "libc/nt/runtime.h"
@@ -30,7 +31,7 @@
static int ttysetcursor(int fd, bool visible) {
struct NtConsoleCursorInfo ntcursor;
char code[8] = "\e[?25l";
if (IsTerminalInarticulate()) return 0;
if (__nocolor) return 0;
if (visible) code[5] = 'h';
if (SupportsWindows()) {
GetConsoleCursorInfo(GetStdHandle(kNtStdOutputHandle), &ntcursor);
+5 -2
View File
@@ -17,14 +17,17 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/tty/tty.h"
#include "libc/bits/safemacros.internal.h"
#include "libc/bits/weaken.h"
#include "libc/calls/calls.h"
#include "libc/calls/termios.h"
#include "libc/dce.h"
#include "libc/errno.h"
#include "libc/fmt/fmt.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/intrin/weaken.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/sock/sock.h"
#include "libc/sock/struct/pollfd.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/poll.h"
#include "libc/sysv/errfuns.h"
+1 -1
View File
@@ -1,7 +1,7 @@
#ifndef COSMOPOLITAN_DSP_TTY_INTERNAL_H_
#define COSMOPOLITAN_DSP_TTY_INTERNAL_H_
#include "dsp/tty/ttyrgb.h"
#include "libc/bits/xmmintrin.internal.h"
#include "libc/intrin/xmmintrin.internal.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
COSMOPOLITAN_C_START_
+2 -2
View File
@@ -17,8 +17,8 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/tty/itoa8.h"
#include "libc/bits/bits.h"
#include "libc/bits/initializer.internal.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/initializer.internal.h"
#include "libc/str/str.h"
struct Itoa8 kItoa8;
+2 -2
View File
@@ -2,8 +2,8 @@
#define DSP_TTY_QUANT_H_
#include "dsp/tty/ttyrgb.h"
#include "libc/assert.h"
#include "libc/bits/bits.h"
#include "libc/bits/xmmintrin.internal.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/xmmintrin.internal.h"
#include "libc/limits.h"
#include "libc/str/str.h"
#if !(__ASSEMBLER__ + __LINKER__ + 0)
+3 -2
View File
@@ -19,7 +19,7 @@
#include "dsp/core/core.h"
#include "dsp/tty/quant.h"
#include "libc/assert.h"
#include "libc/bits/initializer.internal.h"
#include "libc/intrin/initializer.internal.h"
#include "libc/limits.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
@@ -47,7 +47,8 @@ __m128 tty2rgbf_(struct TtyRgb rgbxt) {
static int rgb2xterm256_(int r, int g, int b) {
int cerr, gerr, ir, ig, ib, gray, grai, cr, cg, cb, gv;
gray = round(r * .299 + g * .587 + b * .114);
gray = round(871024 / 4096299. * r + 8788810 / 12288897. * g +
887015 / 12288897. * b);
grai = gray > 238 ? 23 : (gray - 3) / 10;
ir = r < 48 ? 0 : r < 115 ? 1 : (r - 35) / 40;
ig = g < 48 ? 0 : g < 115 ? 1 : (g - 35) / 40;
+1 -1
View File
@@ -17,7 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/tty/quant.h"
#include "libc/bits/xmmintrin.internal.h"
#include "libc/intrin/xmmintrin.internal.h"
struct TtyRgb rgb2ttyf2i_(__m128 rgb) {
__v4si i4;
+1 -1
View File
@@ -17,7 +17,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/tty/tty.h"
#include "libc/alg/arraylist2.internal.h"
#include "libc/mem/arraylist2.internal.h"
#include "libc/runtime/gc.internal.h"
#include "libc/x/x.h"
-7
View File
@@ -31,13 +31,6 @@ int ttysetraw(struct termios *conf, int64_t flags) {
conf->c_cflag &= ~(CSIZE | PARENB);
conf->c_cflag |= CS8;
conf->c_iflag |= IUTF8;
/* if (flags & kTtyLfToCrLf) { */
/* /\* conf->c_oflag &= ~(OLCUC | OCRNL | ONLRET | OFILL | OFDEL); *\/ */
/* /\* conf->c_oflag |= ONLCR | ONOCR; *\/ */
/* conf->c_oflag |= ONLCR; */
/* } else { */
/* conf->c_oflag &= ~OPOST; */
/* } */
if (!(flags & kTtySigs)) {
conf->c_iflag &= ~(IGNBRK | BRKINT);
conf->c_lflag &= ~(ISIG);
+3 -5
View File
@@ -25,7 +25,6 @@ DSP_TTY_A_CHECKS = \
DSP_TTY_A_DIRECTDEPS = \
DSP_CORE \
LIBC_ALG \
LIBC_CALLS \
LIBC_FMT \
LIBC_INTRIN \
@@ -41,8 +40,7 @@ DSP_TTY_A_DIRECTDEPS = \
LIBC_SYSV \
LIBC_TINYMATH \
LIBC_TIME \
LIBC_X \
LIBC_UNICODE
LIBC_X
DSP_TTY_A_DEPS := \
$(call uniq,$(foreach x,$(DSP_TTY_A_DIRECTDEPS),$($(x))))
@@ -55,8 +53,8 @@ $(DSP_TTY_A).pkg: \
$(DSP_TTY_A_OBJS) \
$(foreach x,$(DSP_TTY_A_DIRECTDEPS),$($(x)_A).pkg)
o/$(MODE)/dsp/tty/ttyraster.o: \
OVERRIDE_CFLAGS += \
o/$(MODE)/dsp/tty/ttyraster.o: private \
OVERRIDE_CFLAGS += \
$(MATHEMATICAL)
DSP_TTY_LIBS = $(foreach x,$(DSP_TTY_ARTIFACTS),$($(x)))
+1 -1
View File
@@ -18,7 +18,7 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/tty/tty.h"
#include "libc/alg/alg.h"
#include "libc/mem/alg.h"
#include "libc/nexgen32e/nexgen32e.h"
#include "libc/str/str.h"
+1 -1
View File
@@ -18,7 +18,7 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/tty/itoa8.h"
#include "dsp/tty/tty.h"
#include "libc/bits/safemacros.internal.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/limits.h"
#include "libc/log/check.h"
+1 -1
View File
@@ -18,7 +18,7 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "dsp/tty/internal.h"
#include "dsp/tty/quant.h"
#include "libc/bits/initializer.internal.h"
#include "libc/intrin/initializer.internal.h"
#include "libc/dce.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/stdio.h"
+22 -18
View File
@@ -22,8 +22,8 @@
#include "dsp/tty/ttyrgb.h"
#include "dsp/tty/windex.h"
#include "libc/assert.h"
#include "libc/bits/bits.h"
#include "libc/bits/safemacros.internal.h"
#include "libc/intrin/bits.h"
#include "libc/intrin/safemacros.internal.h"
#include "libc/limits.h"
#include "libc/log/check.h"
#include "libc/log/log.h"
@@ -731,19 +731,19 @@ static bool ChunkEq(struct TtyRgb c[hasatleast 4],
}
static struct TtyRgb *CopyChunk(struct TtyRgb chunk[hasatleast 4],
const struct TtyRgb *c, size_t n) {
chunk[TL] = c[0 + 0];
chunk[TR] = c[0 + 1];
chunk[BL] = c[n + 0];
chunk[BR] = c[n + 1];
const struct TtyRgb *c, size_t xn) {
chunk[TL] = c[00 + 0];
chunk[TR] = c[00 + 1];
chunk[BL] = c[xn + 0];
chunk[BR] = c[xn + 1];
return chunk;
}
static noinline char *CopyRun(char *v, size_t n,
struct TtyRgb lastchunk[hasatleast 4],
const struct TtyRgb **c, size_t *x,
struct TtyRgb *bg, struct TtyRgb *fg,
struct Glyph *glyph) {
static dontinline char *CopyRun(char *v, size_t n,
struct TtyRgb lastchunk[hasatleast 4],
const struct TtyRgb **c, size_t *x,
struct TtyRgb *bg, struct TtyRgb *fg,
struct Glyph *glyph) {
struct TtyRgb chunk[4];
if (memcmp(glyph, &kGlyphs[1][0], sizeof(*glyph)) == 0) {
if (!ttyeq(*bg, *fg)) {
@@ -766,17 +766,21 @@ static noinline char *CopyRun(char *v, size_t n,
/**
* Maps 2×2 pixel chunks onto ANSI UNICODE cells.
* @note h/t Nick Black for his quadrant blitting work on notcurses
* @note yn and xn need to be even
*/
char *ttyraster(char *v, const struct TtyRgb *c, size_t yn, size_t n,
char *ttyraster(char *v, const struct TtyRgb *c, size_t yn, size_t xn,
struct TtyRgb bg, struct TtyRgb fg) {
unsigned y, x;
struct Pick p;
struct Glyph glyph;
struct TtyRgb chun[4], lastchunk[4];
for (y = 0; y < yn; y += 2, c += n) {
if (y) *v++ = '\r', *v++ = '\n';
for (x = 0; x < n; x += 2, c += 2) {
CopyChunk(chun, c, n);
for (y = 0; y < yn; y += 2, c += xn) {
if (y) {
v = stpcpy(v, "\e[0m\r\n");
v = setbgfg(v, bg, fg);
}
for (x = 0; x < xn; x += 2, c += 2) {
CopyChunk(chun, c, xn);
if (ttyquant()->alg == kTtyQuantTrue) {
if (ttyquant()->blocks == kTtyBlocksCp437) {
p = PickBlockCp437True(chun[TL], chun[TR], chun[BL], chun[BR]);
@@ -794,6 +798,6 @@ char *ttyraster(char *v, const struct TtyRgb *c, size_t yn, size_t n,
memcpy(lastchunk, chun, sizeof(chun));
}
}
*v = '\0';
v = stpcpy(v, "\e[0m");
return v;
}
-2
View File
@@ -19,11 +19,9 @@
#include "dsp/tty/tty.h"
#include "libc/assert.h"
#include "libc/calls/calls.h"
#include "libc/calls/sigbits.h"
#include "libc/calls/struct/sigaction.h"
#include "libc/calls/struct/siginfo.h"
#include "libc/calls/termios.h"
#include "libc/calls/typedef/sigaction_f.h"
#include "libc/calls/ucontext.h"
#include "libc/log/log.h"
#include "libc/macros.internal.h"
-1
View File
@@ -17,7 +17,6 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
.source __FILE__
// Returns index of minimum uint16 in array.
//
-1
View File
@@ -17,7 +17,6 @@
│ PERFORMANCE OF THIS SOFTWARE. │
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/macros.internal.h"
.source __FILE__
// Returns index of minimum positive int16 in array.
//
-1
View File
@@ -18,7 +18,6 @@
╚─────────────────────────────────────────────────────────────────────────────*/
#include "libc/nexgen32e/x86feature.h"
#include "libc/macros.internal.h"
.source __FILE__
// Dispatches to fastest windex() implementation.
.initbss 300,_init_windex
+1
View File
@@ -21,6 +21,7 @@
#include "libc/calls/termios.h"
#include "libc/errno.h"
#include "libc/sock/sock.h"
#include "libc/sock/struct/pollfd.h"
#include "libc/sysv/consts/poll.h"
#include "libc/sysv/consts/termios.h"
+8 -5
View File
@@ -10,7 +10,7 @@
#include "libc/log/log.h"
/**
* Automatically launchs GDB Debugger TUI during crash.
* Automatically launches GDB Debugger TUI during crash.
*
* Run the following inside a terminal:
*
@@ -18,9 +18,12 @@
* make -j12 o//examples/auto-launch-gdb-on-crash.com
* o//examples/auto-launch-gdb-on-crash.com
*
* Backtrace is logged instead if run outside interactive terminal.
* Environmental factors such as GDB, MAKEFLAGS, ADDR2LINE, TERM, and
* isatty(STDERR_FILENO) should also be taken into consideration:
* Backtrace is logged instead if run outside interactive terminal. We
* also don't auto-launch GDB on non-Linux since the development tools
* on other platforms generally can't be relied upon to correctly debug
* binaries built with a Linux toolchain. Environmental factors such as
* GDB, MAKEFLAGS, ADDR2LINE, TERM, and isatty(STDERR_FILENO) should
* also be taken into consideration:
*
* $ export GDB=eoatuhshtuone
* $ o//examples/auto-launch-gdb-on-crash.com
@@ -37,7 +40,7 @@
*/
int main(int argc, char *argv[]) {
showcrashreports();
ShowCrashReports();
asm("int3"); /* cf. __die(), perror("msg"), abort(), exit(1), _Exit(1) */
return 0;
}
+62
View File
@@ -0,0 +1,62 @@
#if 0
/*─────────────────────────────────────────────────────────────────╗
│ To the extent possible under law, Justine Tunney has waived │
│ all copyright and related or neighboring rights to this file, │
│ as it is written in the following disclaimers: │
│ • http://unlicense.org/ │
│ • http://creativecommons.org/publicdomain/zero/1.0/ │
╚─────────────────────────────────────────────────────────────────*/
#endif
#include "libc/intrin/bits.h"
#include "libc/dce.h"
#include "libc/log/log.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/stdio.h"
/**
* ASAN static memory safety crash example.
*
* make -j8 MODE=dbg o/dbg/examples/auto-memory-safety-crash.com
* o/dbg/examples/auto-memory-safety-crash.com
*
* You should see:
*
* global redzone 1-byte store at 0x42700d shadow 0x8007ce01
* ./o/dbg/examples/auto-memory-safety-crash.com
* x
* ........................................GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG
* |0 |0 |0 |0 |5 |-18 |-18 |-18 |-18
*  f.☼▼ä     f.☼▼ä     f☼▼D  hello                                                
* 000000400000-000000427000 .text
* 000000427000-000000429000 .data ←address
* 00007fff0000-00008000ffff
* 000080070000-00008008ffff ←shadow
* 0e007fff0000-0e008000ffff
* 100047d20000-100047d3ffff
* 6ffffffe0000-6fffffffffff
* the memory in question belongs to the symbols
* buffer [0x427000,0x42700c] size 13
* the crash was caused by
* 0x00000000004046f3: __die at libc/log/die.c:40
* 0x0000000000404aed: __asan_report_store at libc/intrin/asan.c:1183
* 0x0000000000402552: main at examples/auto-memory-safety-crash.c:27
* 0x000000000040268d: cosmo at libc/runtime/cosmo.S:64
* 0x00000000004021ae: _start at libc/crt/crt.S:77
*
* @see libc/intrin/asancodes.h for meaning of G, etc. and negative numbers
* @see libc/nexgen32e/kcp437.S for meaning of symbols
*/
char buffer[13] = "hello";
int main(int argc, char *argv[]) {
if (!IsAsan()) {
printf("this example is intended for MODE=asan or MODE=dbg\n");
exit(1);
}
ShowCrashReports(); /* not needed but yoinks appropriate symbols */
int i = 13;
asm("" : "+r"(i)); /* prevent compiler being smart */
buffer[i] = 1;
return 0;
}
+74
View File
@@ -0,0 +1,74 @@
#if 0
/*─────────────────────────────────────────────────────────────────╗
│ To the extent possible under law, Justine Tunney has waived │
│ all copyright and related or neighboring rights to this file, │
│ as it is written in the following disclaimers: │
│ • http://unlicense.org/ │
│ • http://creativecommons.org/publicdomain/zero/1.0/ │
╚─────────────────────────────────────────────────────────────────*/
#endif
#include "libc/intrin/bits.h"
#include "libc/dce.h"
#include "libc/log/log.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
/**
* ASAN heap memory safety crash example.
*
* make -j8 MODE=dbg o/dbg/examples/auto-memory-safety-crash2.com
* o/dbg/examples/auto-memory-safety-crash2.com
*
* You should see:
*
* heap overrun 1-byte store at 0x10008004002d shadow 0x20090000005
* ./o/dbg/examples/auto-memory-safety-crash2.com
* x
* OOOOOOOOOOOUUUUUUUUUUUUUUUU.............OOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO
* |-7 |-6 |-6 |0 |5 |-7 |-7 |-7 |-7
*    »!@     ÿ▄:Y╩≥= S       hello ∙∙∙∙∙∙∙           ♪     GT◘&@     á+@     »!@  
* 000000400000-00000042b000 .text
* 00000042b000-00000042d000 .data
* 00007fff0000-00008000ffff
* 000080070000-00008008ffff
* 02008fff0000-02009000ffff ←shadow
* 0e007fff0000-0e008000ffff
* 10003ab90000-10003abaffff
* 100080000000-10008000ffff ←address
* 6ffffffe0000-6fffffffffff
*
* the memory was allocated by
* 0x100080040020 64 bytes [dlmalloc]
* 0x100080040030 13 bytes [actual]
* 402608 main
* 402ba0 cosmo
* 4021af _start
*
* the crash was caused by
* 0x0000000000404793: __die at libc/log/die.c:40
* 0x0000000000404f56: __asan_report_store at libc/intrin/asan.c:1183
* 0x0000000000402579: main at examples/auto-memory-safety-crash2.c:30
* 0x000000000040270f: cosmo at libc/runtime/cosmo.S:64
* 0x00000000004021ae: _start at libc/crt/crt.S:77
*
* @see libc/intrin/asancodes.h for meaning of U, O, etc. and negative numbers
* @see libc/nexgen32e/kcp437.S for meaning of symbols
*/
int main(int argc, char *argv[]) {
if (!IsAsan()) {
printf("this example is intended for MODE=asan or MODE=dbg\n");
exit(1);
}
char *buffer;
ShowCrashReports(); /* not needed but yoinks appropriate symbols */
buffer = malloc(13);
strcpy(buffer, "hello");
int i = 13;
asm("" : "+r"(i)); /* prevent compiler being smart */
buffer[i] = 1;
asm("" : "+r"(buffer)); /* prevent compiler being smart */
return 0;
}
+39
View File
@@ -0,0 +1,39 @@
#if 0
/*─────────────────────────────────────────────────────────────────╗
│ To the extent possible under law, Justine Tunney has waived │
│ all copyright and related or neighboring rights to this file, │
│ as it is written in the following disclaimers: │
│ • http://unlicense.org/ │
│ • http://creativecommons.org/publicdomain/zero/1.0/ │
╚─────────────────────────────────────────────────────────────────*/
#endif
#include "libc/intrin/bits.h"
#include "libc/dce.h"
#include "libc/log/log.h"
#include "libc/mem/mem.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
/**
* ASAN use-after-free memory safety crash example.
*
* make -j8 MODE=dbg o/dbg/examples/auto-memory-safety-crash3.com
* o/dbg/examples/auto-memory-safety-crash3.com
*
*/
int main(int argc, char *argv[]) {
if (!IsAsan()) {
printf("this example is intended for MODE=asan or MODE=dbg\n");
exit(1);
}
char *buffer;
ShowCrashReports(); /* not needed but yoinks appropriate symbols */
buffer = malloc(13);
strcpy(buffer, "hello");
free(buffer);
asm("" : "+r"(buffer)); /* prevent compiler being smart */
buffer[0] = 1;
return 0;
}
+20 -7
View File
@@ -7,17 +7,30 @@
│ • http://creativecommons.org/publicdomain/zero/1.0/ │
╚─────────────────────────────────────────────────────────────────*/
#endif
#include "libc/intrin/kprintf.h"
#include "libc/log/backtrace.internal.h"
#include "libc/log/log.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/stdio.h"
#include "libc/runtime/symbols.internal.h"
int main(int argc, char *argv[]) {
showcrashreports();
ShowCrashReports();
if (IsDebuggerPresent(false)) {
printf("debugger found!\r\n");
DebugBreak();
return 0;
kprintf("debugger found!%n");
} else {
kprintf("try running: gdb %s%n", argv[0]);
kprintf("try running: o//tool/build/strace.com %s%n", argv[0]);
}
printf("try running: gdb %s\r\n", argv[0]);
return 1;
asm volatile("mov\t%4,%%r10\n\t"
"mov\t%5,%%r8\n\t"
"mov\t%6,%%r9\n\t"
"int3"
: /* no outputs */
: "a"(0), "D"(1), "S"(2), "d"(3), "g"(4), "g"(5), "g"(6)
: "r8", "r9", "r10");
printf("recovered from SIGTRAP without handler\r\n");
return 0;
}
+1 -1
View File
@@ -35,7 +35,7 @@ STATIC_YOINK("ssl_root_support");
#define DFL_FILENAME "cert.crt"
#define DFL_CA_FILE ""
#define DFL_CRL_FILE ""
#define DFL_CA_PATH "zip:usr/share/ssl/root"
#define DFL_CA_PATH "/zip/usr/share/ssl/root"
#define DFL_SERVER_NAME "localhost"
#define DFL_SERVER_PORT "4433"
#define DFL_DEBUG_LEVEL 0
+88
View File
@@ -0,0 +1,88 @@
#if 0
/*─────────────────────────────────────────────────────────────────╗
│ To the extent possible under law, Justine Tunney has waived │
│ all copyright and related or neighboring rights to this file, │
│ as it is written in the following disclaimers: │
│ • http://unlicense.org/ │
│ • http://creativecommons.org/publicdomain/zero/1.0/ │
╚─────────────────────────────────────────────────────────────────*/
#endif
#include "libc/log/check.h"
/**
* @fileoverview Check Macros
*
* The simplest assertion is:
*
* assert(123 == x);
*
* This has some downsides:
*
* 1. It's nice to know what `x` is when it crashes
* 2. It's sometimes nice to have the check always take effect.
* 3. It's problematic that assert() can't do __builtin_assume()
*
* Cosmopolitan provides alternative macros like:
*
* - `CHECK(EXPR, ...)`
* - `CHECK_EQ(WANT, GOT, ...)`
* - `CHECK_NE(WANT, GOT, ...)`
* - `CHECK_GT(WANT, GOT, ...)`
* - `CHECK_LT(WANT, GOT, ...)`
* - `CHECK_NOTNULL(EXPR, ...)`
*
* The CHECK macros always happen. They always kill the process when
* they fail. Printf formatting information may be provided as extra
* arguments. On the other hand, there exists:
*
* - `DCHECK(EXPR, ...)`
* - `DCHECK_EQ(WANT, GOT, ...)`
* - `DCHECK_NE(WANT, GOT, ...)`
* - `DCHECK_GT(WANT, GOT, ...)`
* - `DCHECK_LT(WANT, GOT, ...)`
* - `DCHECK_NOTNULL(EXPR, ...)`
*
* The DCHECK macros always happen when NDEBUG isn't defined. When
* NDEBUG is defined, they still happen, but in a special way that
* causes the compiler to recognize their failure as undefined behavior.
* What this means is that, if the provided expressions are pure without
* side-effects, then the code compiles down to nothing and the compiler
* may be able to use the knowledge of something being the case in order
* to optimize other code adjacent to your DCHECK.
*
* In the default build modes, this prints lots of information:
*
* error:examples/check.c:23:check.com: check failed on nightmare pid 15412
* CHECK_EQ(123, some_source_code);
* → 0x7b (123)
* == 0x64 (some_source_code)
* extra info: hello
* EUNKNOWN/0/No error information
* ./o//examples/check.com
* 0x0000000000407404: __die at libc/log/die.c:42
* 0x0000000000407340: __check_fail at libc/log/checkfail.c:73
* 0x00000000004071d0: main at examples/check.c:23
* 0x000000000040256e: cosmo at libc/runtime/cosmo.S:69
* 0x000000000040217d: _start at libc/crt/crt.S:85
*
* In NDEBUG mode (e.g. MODE=rel, MODE=tiny, etc.) this prints a much
* simpler message that, most importantly, doesn't include any source
* code, although it still includes the file name for reference.
*
* error:examples/check.c:14: check failed: 123 == 100: extra info: hello
*
* That way your release binaries won't leak CI. You may optionally link
* the following functions to further expand the information shown by
* the NDEBUG check failure:
*
* STATIC_YOINK("__die");
* STATIC_YOINK("strerror");
*
* Please note that backtraces aren't ever available in MODE=tiny.
*/
int main(int argc, char *argv[]) {
int some_source_code = 100;
CHECK_EQ(123, some_source_code, "extra info: %s", "hello");
return 0;
}
+192
View File
@@ -0,0 +1,192 @@
#if 0
/*─────────────────────────────────────────────────────────────────╗
│ To the extent possible under law, Justine Tunney has waived │
│ all copyright and related or neighboring rights to this file, │
│ as it is written in the following disclaimers: │
│ • http://unlicense.org/ │
│ • http://creativecommons.org/publicdomain/zero/1.0/ │
╚─────────────────────────────────────────────────────────────────*/
#endif
#include "libc/assert.h"
#include "libc/errno.h"
#include "libc/fmt/conv.h"
#include "libc/log/check.h"
#include "libc/mem/mem.h"
#include "libc/runtime/gc.internal.h"
#include "libc/runtime/runtime.h"
#include "libc/stdio/stdio.h"
#include "libc/str/str.h"
#include "libc/sysv/consts/ex.h"
#include "libc/sysv/consts/exit.h"
#include "third_party/getopt/getopt.h"
#include "third_party/zlib/zlib.h"
#define CHUNK 32768
#define USAGE \
" <STDIN >STDOUT\n\
\n\
SYNOPSIS\n\
\n\
Zlib Compressor\n\
\n\
FLAGS\n\
\n\
-?\n\
-h help\n\
-0 disable compression\n\
-1 fastest compression\n\
-4 coolest compression\n\
-9 maximum compression\n\
-F fixed strategy (advanced)\n\
-L filtered strategy (advanced)\n\
-R run length strategy (advanced)\n\
-H huffman only strategy (advanced)\n\
\n"
// clang-format off
// make -j8 o//examples && dd if=/dev/urandom count=100 | tee a | o//examples/compress.com | o//examples/decompress.com >b && sha1sum a b
/*
# data file is o/dbg/third_party/python/python.com
# level 0 147517 compress 495 MB/s decompress 1.4 GB/s
# level 1 80274 compress 29.2 MB/s decompress 303 MB/s
# level 2 79384 compress 33.8 MB/s decompress 212 MB/s
# level 3 78875 compress 28.9 MB/s decompress 224 MB/s
# level 4 78010 compress 27.1 MB/s decompress 319 MB/s <-- sweet spot?
# level 5 77107 compress 19.5 MB/s decompress 273 MB/s
# level 6 75081 compress 10.0 MB/s decompress 99.3 MB/s
# level 7 75022 compress 7.5 MB/s decompress 287 MB/s
# level 8 75016 compress 5.4 MB/s decompress 109 MB/s
# level 9 75016 compress 5.4 MB/s decompress 344 MB/s
m=
make -j8 MODE=$m o/$m/examples || exit
for level in $(seq 1 9); do
for strategy in F L R H; do
o/$m/examples/compress.com -$strategy$level <o/dbg/third_party/python/python.com | dd count=10000 2>/tmp/info >/tmp/comp
compspeed=$(grep -Po '[.\d]+ \w+/s' /tmp/info)
o/$m/examples/decompress.com </tmp/comp | dd count=10000 2>/tmp/info >/dev/null
decompspeed=$(grep -Po '[.\d]+ \w+/s' /tmp/info)
size=$(o/$m/examples/compress.com -$strategy$level <o/$m/examples/compress.com | wc -c)
echo "level $strategy $level $size compress $compspeed decompress $decompspeed"
done
done
*/
// clang-format on
int level;
int strategy;
wontreturn void PrintUsage(int rc, FILE *f) {
fputs("usage: ", f);
fputs(program_invocation_name, f);
fputs(USAGE, f);
exit(rc);
}
void GetOpts(int argc, char *argv[]) {
int opt;
while ((opt = getopt(argc, argv, "?hFLRH0123456789")) != -1) {
switch (opt) {
case 'F':
strategy = Z_FIXED;
break;
case 'L':
strategy = Z_FILTERED;
break;
case 'R':
strategy = Z_RLE;
break;
case 'H':
strategy = Z_HUFFMAN_ONLY;
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
level = opt - '0';
break;
case 'h':
case '?':
PrintUsage(EXIT_SUCCESS, stdout);
default:
PrintUsage(EX_USAGE, stderr);
}
}
}
int compressor(int infd, int outfd) {
z_stream zs;
int rc, flush;
unsigned have;
unsigned char *inbuf;
unsigned char *outbuf;
inbuf = gc(valloc(CHUNK));
outbuf = gc(valloc(CHUNK));
zs.zalloc = 0;
zs.zfree = 0;
zs.opaque = 0;
rc = deflateInit2(&zs, level, Z_DEFLATED, MAX_WBITS, DEF_MEM_LEVEL, strategy);
if (rc != Z_OK) return rc;
do {
rc = read(infd, inbuf, CHUNK);
if (rc == -1) {
deflateEnd(&zs);
return Z_ERRNO;
}
zs.avail_in = rc;
flush = !rc ? Z_FINISH : Z_SYNC_FLUSH;
zs.next_in = inbuf;
do {
zs.avail_out = CHUNK;
zs.next_out = outbuf;
rc = deflate(&zs, flush);
assert(rc != Z_STREAM_ERROR);
have = CHUNK - zs.avail_out;
if (write(outfd, outbuf, have) != have) {
deflateEnd(&zs);
return Z_ERRNO;
}
} while (!zs.avail_out);
assert(!zs.avail_in);
} while (flush != Z_FINISH);
assert(rc == Z_STREAM_END);
deflateEnd(&zs);
return Z_OK;
}
const char *zerr(int rc) {
switch (rc) {
case Z_ERRNO:
return strerror(errno);
case Z_STREAM_ERROR:
return "invalid compression level";
case Z_DATA_ERROR:
return "invalid or incomplete deflate data";
case Z_MEM_ERROR:
return "out of memory";
case Z_VERSION_ERROR:
return "zlib version mismatch!";
default:
unreachable;
}
}
int main(int argc, char *argv[]) {
int rc;
level = Z_DEFAULT_COMPRESSION;
strategy = Z_DEFAULT_STRATEGY;
GetOpts(argc, argv);
rc = compressor(0, 1);
if (rc == Z_OK) {
return 0;
} else {
fprintf(stderr, "error: compressor: %s\n", zerr(rc));
return 1;
}
}
Loaded 100 of 13629 files, more files were not shown because too many files have changed in this diff. Show more