diff --git a/CHANGELOG.md b/CHANGELOG.md index 847292de..f2e477dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,30 @@ +# v6.24.0 +- [#3671](https://github.com/xmrig/xmrig/pull/3671) Fixed detection of L2 cache size for some complex NUMA topologies. +- [#3674](https://github.com/xmrig/xmrig/pull/3674) Fixed ARMv7 build. +- [#3677](https://github.com/xmrig/xmrig/pull/3677) Fixed auto-config for AMD CPUs with less than 2 MB L3 cache per thread. +- [#3678](https://github.com/xmrig/xmrig/pull/3678) Improved IPv6 support: the new default settings use IPv6 equally with IPv4. + +# v6.23.0 +- [#3668](https://github.com/xmrig/xmrig/issues/3668) Added support for Windows ARM64. +- [#3665](https://github.com/xmrig/xmrig/pull/3665) Tweaked auto-config for AMD CPUs with < 2 MB L3 cache per thread. + +# v6.22.3 +- [#3605](https://github.com/xmrig/xmrig/pull/3605) CUDA backend: added missing RandomX dataset update. +- [#3646](https://github.com/xmrig/xmrig/pull/3646) Optimized auto-config for AMD CPUs with less than 2 MB L3 cache per thread. +- [#3652](https://github.com/xmrig/xmrig/pull/3652) Fixed possible crash when submitting RandomX benchmark. +- [#3662](https://github.com/xmrig/xmrig/pull/3662) Fixed OpenCL kernel compilation error on some platforms. + +# v6.22.2 +- [#3569](https://github.com/xmrig/xmrig/pull/3569) Fixed corrupted API output in some rare conditions. +- [#3571](https://github.com/xmrig/xmrig/pull/3571) Fixed number of threads on the new Intel Core Ultra CPUs. + +# v6.22.1 +- [#3531](https://github.com/xmrig/xmrig/pull/3531) Always reset nonce on RandomX dataset change. +- [#3534](https://github.com/xmrig/xmrig/pull/3534) Fixed threads auto-config on Zen5. +- [#3535](https://github.com/xmrig/xmrig/pull/3535) RandomX: tweaks for Zen5. +- [#3539](https://github.com/xmrig/xmrig/pull/3539) Added Zen5 to `randomx_boost.sh`. +- [#3540](https://github.com/xmrig/xmrig/pull/3540) Detect AMD engineering samples in `randomx_boost.sh`. + # v6.22.0 - [#2411](https://github.com/xmrig/xmrig/pull/2411) Added support for [Yada](https://yadacoin.io/) (`rx/yada` algorithm). - [#3492](https://github.com/xmrig/xmrig/pull/3492) Fixed `--background` option on Unix systems. diff --git a/CMakeLists.txt b/CMakeLists.txt index 70d57d10..31392322 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.10) project(xmrig) option(WITH_HWLOC "Enable hwloc support" ON) @@ -240,7 +240,10 @@ add_executable(${CMAKE_PROJECT_NAME} ${HEADERS} ${SOURCES} ${SOURCES_OS} ${HEADE target_link_libraries(${CMAKE_PROJECT_NAME} ${XMRIG_ASM_LIBRARY} ${OPENSSL_LIBRARIES} ${UV_LIBRARIES} ${EXTRA_LIBS} ${CPUID_LIB} ${ARGON2_LIBRARY} ${ETHASH_LIBRARY} ${GHOSTRIDER_LIBRARY}) if (WIN32) - add_custom_command(TARGET ${CMAKE_PROJECT_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/bin/WinRing0/WinRing0x64.sys" $) + if (NOT ARM_TARGET) + add_custom_command(TARGET ${CMAKE_PROJECT_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/bin/WinRing0/WinRing0x64.sys" $) + endif() + add_custom_command(TARGET ${CMAKE_PROJECT_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/scripts/benchmark_1M.cmd" $) add_custom_command(TARGET ${CMAKE_PROJECT_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/scripts/benchmark_10M.cmd" $) add_custom_command(TARGET ${CMAKE_PROJECT_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_SOURCE_DIR}/scripts/pool_mine_example.cmd" $) @@ -249,5 +252,5 @@ if (WIN32) endif() if (CMAKE_CXX_COMPILER_ID MATCHES Clang AND CMAKE_BUILD_TYPE STREQUAL Release AND NOT CMAKE_GENERATOR STREQUAL Xcode) - add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD COMMAND ${CMAKE_STRIP} ${CMAKE_PROJECT_NAME}) + add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD COMMAND ${CMAKE_STRIP} "$") endif() diff --git a/cmake/cpu.cmake b/cmake/cpu.cmake index cdc35f10..12dbe9b1 100644 --- a/cmake/cpu.cmake +++ b/cmake/cpu.cmake @@ -29,6 +29,8 @@ else() set(WITH_VAES OFF) endif() +add_definitions(-DRAPIDJSON_WRITE_DEFAULT_FLAGS=6) # rapidjson::kWriteNanAndInfFlag | rapidjson::kWriteNanAndInfNullFlag + if (ARM_V8) set(ARM_TARGET 8) elseif (ARM_V7) @@ -36,7 +38,7 @@ elseif (ARM_V7) endif() if (NOT ARM_TARGET) - if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|armv8-a)$") + if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64|armv8-a)$") set(ARM_TARGET 8) elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv7|armv7f|armv7s|armv7k|armv7-a|armv7l|armv7ve)$") set(ARM_TARGET 7) @@ -49,7 +51,7 @@ if (ARM_TARGET AND ARM_TARGET GREATER 6) message(STATUS "Use ARM_TARGET=${ARM_TARGET} (${CMAKE_SYSTEM_PROCESSOR})") - if (ARM_TARGET EQUAL 8) + if (ARM_TARGET EQUAL 8 AND (CMAKE_CXX_COMPILER_ID MATCHES GNU OR CMAKE_CXX_COMPILER_ID MATCHES Clang)) CHECK_CXX_COMPILER_FLAG(-march=armv8-a+crypto XMRIG_ARM_CRYPTO) if (XMRIG_ARM_CRYPTO) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index f55ddcdc..9abf212a 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -26,7 +26,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES GNU) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ARM8_CXX_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARM8_CXX_FLAGS} -flax-vector-conversions") elseif (ARM_TARGET EQUAL 7) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv7-a -mfpu=neon") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv7-a -mfpu=neon -flax-vector-conversions") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv7-a -mfpu=neon -flax-vector-conversions") else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes") @@ -63,10 +63,10 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES MSVC) elseif (CMAKE_CXX_COMPILER_ID MATCHES Clang) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") - set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -Ofast -funroll-loops -fmerge-all-constants") + set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -funroll-loops -fmerge-all-constants") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -fexceptions -fno-rtti -Wno-missing-braces") - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Ofast -funroll-loops -fmerge-all-constants") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -fexceptions -fno-rtti") + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -funroll-loops -fmerge-all-constants") if (ARM_TARGET EQUAL 8) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ARM8_CXX_FLAGS}") @@ -84,10 +84,9 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES Clang) endif() endif() - if (BUILD_STATIC) + if ((WIN32 AND ARM_TARGET) OR BUILD_STATIC) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static") endif() - endif() if (NOT WIN32) diff --git a/cmake/os.cmake b/cmake/os.cmake index 19b208b3..8f70e9f4 100644 --- a/cmake/os.cmake +++ b/cmake/os.cmake @@ -20,7 +20,6 @@ else() endif() endif() - if (XMRIG_OS_WIN) add_definitions(-DWIN32 -DXMRIG_OS_WIN) elseif(XMRIG_OS_APPLE) diff --git a/cmake/randomx.cmake b/cmake/randomx.cmake index 2d54dbf7..a50e078f 100644 --- a/cmake/randomx.cmake +++ b/cmake/randomx.cmake @@ -1,4 +1,18 @@ if (WITH_RANDOMX) + include(CheckSymbolExists) + + if (WIN32) + check_symbol_exists(_aligned_malloc "stdlib.h" HAVE_ALIGNED_MALLOC) + if (HAVE_ALIGNED_MALLOC) + add_compile_definitions(HAVE_ALIGNED_MALLOC) + endif() + else() + check_symbol_exists(posix_memalign "stdlib.h" HAVE_POSIX_MEMALIGN) + if (HAVE_POSIX_MEMALIGN) + add_compile_definitions(HAVE_POSIX_MEMALIGN) + endif() + endif() + add_definitions(/DXMRIG_ALGO_RANDOMX) set(WITH_ARGON2 ON) diff --git a/scripts/build.hwloc.sh b/scripts/build.hwloc.sh index db85fb6f..f4a2c41a 100755 --- a/scripts/build.hwloc.sh +++ b/scripts/build.hwloc.sh @@ -1,8 +1,8 @@ #!/bin/sh -e HWLOC_VERSION_MAJOR="2" -HWLOC_VERSION_MINOR="10" -HWLOC_VERSION_PATCH="0" +HWLOC_VERSION_MINOR="12" +HWLOC_VERSION_PATCH="1" HWLOC_VERSION="${HWLOC_VERSION_MAJOR}.${HWLOC_VERSION_MINOR}.${HWLOC_VERSION_PATCH}" diff --git a/scripts/build.openssl3.sh b/scripts/build.openssl3.sh index a935712e..4dd00382 100755 --- a/scripts/build.openssl3.sh +++ b/scripts/build.openssl3.sh @@ -1,6 +1,6 @@ #!/bin/sh -e -OPENSSL_VERSION="3.0.14" +OPENSSL_VERSION="3.0.16" mkdir -p deps mkdir -p deps/include diff --git a/scripts/build.uv.sh b/scripts/build.uv.sh index ca052f7a..47587cec 100755 --- a/scripts/build.uv.sh +++ b/scripts/build.uv.sh @@ -1,6 +1,6 @@ #!/bin/sh -e -UV_VERSION="1.48.0" +UV_VERSION="1.51.0" mkdir -p deps mkdir -p deps/include diff --git a/scripts/generate_cl.js b/scripts/generate_cl.js index ea22cb48..287eefc7 100644 --- a/scripts/generate_cl.js +++ b/scripts/generate_cl.js @@ -6,7 +6,6 @@ const fs = require('fs'); const path = require('path'); const { text2h, text2h_bundle, addIncludes } = require('./js/opencl'); const { opencl_minify } = require('./js/opencl_minify'); -const cwd = process.cwd(); function cn() @@ -76,18 +75,24 @@ function kawpow() fs.writeFileSync('kawpow_dag_cl.h', text2h(kawpow_dag, 'xmrig', 'kawpow_dag_cl')); } +for (let i = 0; i < 2; i++) { + if (fs.existsSync('src/backend/opencl/cl/OclSource.h')) { + break; + } -process.chdir(path.resolve('src/backend/opencl/cl/cn')); + process.chdir('..'); +} +process.chdir(path.resolve('src/backend/opencl/cl')); + +const cwd = process.cwd(); + +process.chdir(path.resolve(cwd, 'cn')); cn(); cn_r(); -process.chdir(cwd); -process.chdir(path.resolve('src/backend/opencl/cl/rx')); - +process.chdir(path.resolve(cwd, 'rx')); rx(); -process.chdir(cwd); -process.chdir(path.resolve('src/backend/opencl/cl/kawpow')); - +process.chdir(path.resolve(cwd, 'kawpow')); kawpow(); diff --git a/scripts/randomx_boost.sh b/scripts/randomx_boost.sh index adfc4bdf..8580229a 100755 --- a/scripts/randomx_boost.sh +++ b/scripts/randomx_boost.sh @@ -8,7 +8,7 @@ else modprobe msr allow_writes=on fi -if grep -E 'AMD Ryzen|AMD EPYC' /proc/cpuinfo > /dev/null; +if grep -E 'AMD Ryzen|AMD EPYC|AuthenticAMD' /proc/cpuinfo > /dev/null; then if grep "cpu family[[:space:]]\{1,\}:[[:space:]]25" /proc/cpuinfo > /dev/null; then @@ -28,6 +28,14 @@ if grep -E 'AMD Ryzen|AMD EPYC' /proc/cpuinfo > /dev/null; wrmsr -a 0xc001102b 0x2000cc10 echo "MSR register values for Zen3 applied" fi + elif grep "cpu family[[:space:]]\{1,\}:[[:space:]]26" /proc/cpuinfo > /dev/null; + then + echo "Detected Zen5 CPU" + wrmsr -a 0xc0011020 0x4400000000000 + wrmsr -a 0xc0011021 0x4000000000040 + wrmsr -a 0xc0011022 0x8680000401570000 + wrmsr -a 0xc001102b 0x2040cc10 + echo "MSR register values for Zen5 applied" else echo "Detected Zen1/Zen2 CPU" wrmsr -a 0xc0011020 0 diff --git a/src/3rdparty/argon2/CMakeLists.txt b/src/3rdparty/argon2/CMakeLists.txt index 1ad977f0..a9751fd9 100644 --- a/src/3rdparty/argon2/CMakeLists.txt +++ b/src/3rdparty/argon2/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.10) project(argon2 C) set(CMAKE_C_STANDARD 99) diff --git a/src/3rdparty/hwloc/CMakeLists.txt b/src/3rdparty/hwloc/CMakeLists.txt index 37b88cbb..80f97920 100644 --- a/src/3rdparty/hwloc/CMakeLists.txt +++ b/src/3rdparty/hwloc/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.10) project (hwloc C) include_directories(include) diff --git a/src/3rdparty/hwloc/NEWS b/src/3rdparty/hwloc/NEWS index 62cc687e..cf12c5e4 100644 --- a/src/3rdparty/hwloc/NEWS +++ b/src/3rdparty/hwloc/NEWS @@ -1,5 +1,5 @@ Copyright © 2009 CNRS -Copyright © 2009-2023 Inria. All rights reserved. +Copyright © 2009-2025 Inria. All rights reserved. Copyright © 2009-2013 Université Bordeaux Copyright © 2009-2011 Cisco Systems, Inc. All rights reserved. Copyright © 2020 Hewlett Packard Enterprise. All rights reserved. @@ -17,6 +17,117 @@ bug fixes (and other actions) for each version of hwloc since version 0.9. +Version 2.12.1 +-------------- +* Add hwloc-calc's --default-nodes option to hwloc-bind and hwloc-info. +* Improve the --best-memattr "default" fallback, try to use "default" + memory nodes, and add verbose messages and warnings if some + performance info are incomplete or missing. + Thanks to Antoine Morvan for the report. +* Fix CPU and memory binding on different locations, + thanks to Antoine Morvan for the report. +* Add HWLOC_LOCAL_NUMANODE_FLAG_INTERSECT_LOCALITY and enable it by + default in hwloc-calc --local-memory for finding local NUMA nodes + that do not exactly match input locations. + Thanks to Antoine Morvan for the report. +* Fix a possible crash in the x86 backend when Qemu is configured to + expose multicore/thread CPUs that are actually single-core/thread. + Thanks to Georg Pfuetzenreuter. + + +Version 2.12.0 +-------------- +* Add hwloc_topology_get_default_nodeset() for the set of default + NUMA nodes. + - hwloc-calc now has --default-nodes option. +* Rework oneAPI LevelZero support to use zesInit() and avoid the need + to set ZES_ENABLE_SYSMAN=1 in the environment. + - zesDriverGetDeviceByUuidExp() is now required in the L0 runtime. + - ZES/Sysman variants were added in hwloc/levelzero.h to specifically + handle ZES/Sysman device handles. +* Fix the locality of AMD GPU partitions, thanks to Edgar Leon for + reporting and debugging the issue. +* Better detect Cray Slingshot NICs, thanks to Edgar Leon. +* Add support for Die objects and Module groups on Windows. +* Only filter-out Dies that are identical to their Packages + when it applies to all Dies. +* Improve hwloc-calc to handle CPU-less NUMA nodes or platforms with + heterogeneous memory without requiring --nodeset-output. +* hwloc-calc now accepts counting/listing cpukinds and memory tiers + with -N and -I cpukind/memorytier. +* The systemd-dbus-api output of hwloc-calc has changed, and + --nodeset-output-format was added, to support NUMA node outputs. + Thanks to Pierre Neyron. +* Update NVLink bandwidth and CUDA capabilities up to NVIDIA Blackwell. +* Fix some NUMA syscalls on Linux for platforms with old libc headers. +* Some minor fixes in distances. + + +Version 2.11.2 +-------------- +* Add missing CPU info attrs on aarch64 on Linux. +* Use ACPI CPPC on Linux to get better information about cpukinds, + at least on AMD CPUs. +* Fix crash when manipulating cpukinds after topology + duplication, thanks to Hadrien Grasland for the report. +* Fix missing input target checks in memattr functions, + thanks to Hadrien Grasland for the report. +* Fix a memory leak when ignoring NUMA distances on FreeBSD. +* Fix build failure on old Linux distributions without accessat(). +* Fix non-Windows importing of XML topologies and CPUID dumps exported + on Windows. +* hwloc-calc --cpuset-output-format systemd-dbus-api now allows + to generate AllowedCPUs information for systemd slices. + See the hwloc-calc manpage for examples. Thanks to Pierre Neyron. +* Some fixes in manpage EXAMPLES and split them into subsections. + + +Version 2.11.1 +-------------- +* Fix bash completions, thanks Tavis Rudd. + + +Version 2.11.0 +-------------- +* API + + Add HWLOC_MEMBIND_WEIGHTED_INTERLEAVE memory binding policy on + Linux 6.9+. Thanks to Honggyu Kim for the patch. + - weighted_interleave_membind is added to membind support bits. + - The "weighted" policy is added to the hwloc-bind tool. + + Add hwloc_obj_set_subtype(). Thanks to Hadrien Grasland for the report. +* GPU support + + Don't hide the GPU NUMA node on NVIDIA Grace Hopper. + + Get Intel GPU OpenCL device locality. + + Add bandwidths between subdevices in the LevelZero XeLinkBandwidth + matrix. + + Fix PCI Gen4+ link speed of NVIDIA GPU obtained from NVML, + thanks to Akram Sbaih for the report. +* Windows support + + Fix Windows support when UNICODE is enabled, several hwloc features + were missing, thanks to Martin for the report. + + Fix the enabling of CUDA in Windows CMake build, + Thanks to Moritz Kreutzer for the patch. + + Fix CUDA/OpenCL test source path in Windows CMake. +* Tools + + Option --best-memattr may now return multiple nodes. Additional + configuration flags may be given to tweak its behavior. + + hwloc-info has a new --get-attr option to get a single attribute. + + hwloc-info now supports "levels", "support" and "topology" + special keywords for backward compatibility for hwloc 3.0. + + The --taskset command-line option is superseded by the new + --cpuset-output-format which also allows to export as list. + + hwloc-calc may now import bitmasks described as a list of bits + with the new "--cpuset-input-format list". +* Misc + + The MemoryTiersNr info attribute in the root object now says how many + memory tiers were built. Thanks to Antoine Morvan for the report. + + Fix the management of infinite cpusets in the bitmap printf/sscanf + API as well as in command-line tools. + + Add section "Compiling software on top of hwloc's C API" in the + documentation with examples for GNU Make and CMake, + thanks to Florent Pruvost for the help. + + Version 2.10.0 -------------- * Heterogeneous Memory core improvements diff --git a/src/3rdparty/hwloc/README b/src/3rdparty/hwloc/README index f2971d07..91fe2066 100644 --- a/src/3rdparty/hwloc/README +++ b/src/3rdparty/hwloc/README @@ -418,14 +418,8 @@ return 0; } hwloc provides a pkg-config executable to obtain relevant compiler and linker -flags. For example, it can be used thusly to compile applications that utilize -the hwloc library (assuming GNU Make): - -CFLAGS += $(shell pkg-config --cflags hwloc) -LDLIBS += $(shell pkg-config --libs hwloc) - -hwloc-hello: hwloc-hello.c - $(CC) hwloc-hello.c $(CFLAGS) -o hwloc-hello $(LDLIBS) +flags. See Compiling software on top of hwloc's C API for details on building +program on top of hwloc's API using GNU Make or CMake. On a machine 2 processor packages -- each package of which has two processing cores -- the output from running hwloc-hello could be something like the diff --git a/src/3rdparty/hwloc/VERSION b/src/3rdparty/hwloc/VERSION index cd608187..7f6675cb 100644 --- a/src/3rdparty/hwloc/VERSION +++ b/src/3rdparty/hwloc/VERSION @@ -8,8 +8,8 @@ # Please update HWLOC_VERSION* in contrib/windows/hwloc_config.h too. major=2 -minor=10 -release=0 +minor=12 +release=1 # greek is used for alpha or beta release tags. If it is non-empty, # it will be appended to the version number. It does not have to be @@ -22,7 +22,7 @@ greek= # The date when this release was created -date="Dec 04, 2023" +date="May 12, 2025" # If snapshot=1, then use the value from snapshot_version as the # entire hwloc version (i.e., ignore major, minor, release, and @@ -41,6 +41,6 @@ snapshot_version=${major}.${minor}.${release}${greek}-git # 2. Version numbers are described in the Libtool current:revision:age # format. -libhwloc_so_version=22:0:7 +libhwloc_so_version=25:0:10 # Please also update the lines in contrib/windows/libhwloc.vcxproj diff --git a/src/3rdparty/hwloc/include/hwloc.h b/src/3rdparty/hwloc/include/hwloc.h index 18ea1dfa..bcae3257 100644 --- a/src/3rdparty/hwloc/include/hwloc.h +++ b/src/3rdparty/hwloc/include/hwloc.h @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2022 Inria. All rights reserved. + * Copyright © 2009-2025 Inria. All rights reserved. * Copyright © 2009-2012 Université Bordeaux * Copyright © 2009-2020 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -77,6 +77,25 @@ extern "C" { #endif +/** \defgroup hwlocality_api_error_reporting Error reporting in the API + * @{ + * Most functions in the hwloc API return an integer value. + * Unless documentated differently, they return 0 on success + * and -1 on error. + * Functions that return a pointer type return \c NULL on error. + * + * \p errno will be set to a meaningful value whenever possible. + * This includes the usual \c EINVAL when invalid function parameters are passed + * or \c ENOMEM when an internal allocation fails. + * Some specific \c errno value are also used, for instance for binding + * errors as documented in \ref hwlocality_cpubinding. + * + * Some modules describe return values of their functions + * in their introduction, for instance in \ref hwlocality_bitmap. + * @} + */ + + /** \defgroup hwlocality_api_version API version * @{ */ @@ -93,11 +112,13 @@ extern "C" { * Two stable releases of the same series usually have the same ::HWLOC_API_VERSION * even if their HWLOC_VERSION are different. */ -#define HWLOC_API_VERSION 0x00020800 +#define HWLOC_API_VERSION 0x00020c00 /** \brief Indicate at runtime which hwloc API version was used at build time. * * Should be ::HWLOC_API_VERSION if running on the same version. + * + * \return the build-time version number. */ HWLOC_DECLSPEC unsigned hwloc_get_api_version(void); @@ -242,6 +263,11 @@ typedef enum { * This is the smallest object representing Memory resources, * it cannot have any child except Misc objects. * However it may have Memory-side cache parents. + * + * NUMA nodes may correspond to different kinds of memory + * (DRAM, HBM, CXL-DRAM, etc.). When hwloc is able to guess + * that kind, it is specified in the subtype field of the object. + * See also \ref attributes_normal in the main documentation. * * There is always at least one such object in the topology * even if the machine is not NUMA. @@ -317,6 +343,13 @@ typedef enum { HWLOC_OBJ_DIE, /**< \brief Die within a physical package. * A subpart of the physical package, that contains multiple cores. + * + * Some operating systems (e.g. Linux) may expose a single die per package + * even if the hardware does not support dies at all. To avoid showing + * such non-existing dies, hwloc will filter them out if all of them are + * identical to packages. + * This is functionally equivalent to ::HWLOC_TYPE_FILTER_KEEP_STRUCTURE + * being enforced for Dies versus Packages. */ HWLOC_OBJ_TYPE_MAX /**< \private Sentinel value */ @@ -358,14 +391,17 @@ typedef enum hwloc_obj_osdev_type_e { /** \brief Compare the depth of two object types * * Types shouldn't be compared as they are, since newer ones may be added in - * the future. This function returns less than, equal to, or greater than zero - * respectively if \p type1 objects usually include \p type2 objects, are the - * same as \p type2 objects, or are included in \p type2 objects. If the types - * can not be compared (because neither is usually contained in the other), - * ::HWLOC_TYPE_UNORDERED is returned. Object types containing CPUs can always - * be compared (usually, a system contains machines which contain nodes which - * contain packages which contain caches, which contain cores, which contain - * processors). + * the future. + * + * \return A negative integer if \p type1 objects usually include \p type2 objects. + * \return A positive integer if \p type1 objects are usually included in \p type2 objects. + * \return 0 if \p type1 and \p type2 objects are the same. + * \return ::HWLOC_TYPE_UNORDERED if objects cannot be compared + * (because neither is usually contained in the other). + * + * \note Object types containing CPUs can always be compared + * (usually, a machine contains packages, which contain caches, + * which contain cores, which contain PUs). * * \note ::HWLOC_OBJ_PU will always be the deepest, * while ::HWLOC_OBJ_MACHINE is always the highest. @@ -575,7 +611,7 @@ struct hwloc_obj { * \note Its value must not be changed, hwloc_bitmap_dup() must be used instead. */ - struct hwloc_info_s *infos; /**< \brief Array of stringified info type=name. */ + struct hwloc_info_s *infos; /**< \brief Array of info attributes (name and value strings). */ unsigned infos_count; /**< \brief Size of infos array. */ /* misc */ @@ -632,33 +668,48 @@ union hwloc_obj_attr_u { /** \brief PCI Device specific Object Attributes */ struct hwloc_pcidev_attr_s { #ifndef HWLOC_HAVE_32BITS_PCI_DOMAIN - unsigned short domain; /* Only 16bits PCI domains are supported by default */ + unsigned short domain; /**< \brief Domain number (xxxx in the PCI BDF notation xxxx:yy:zz.t). + * Only 16bits PCI domains are supported by default. */ #else - unsigned int domain; /* 32bits PCI domain support break the library ABI, hence it's disabled by default */ + unsigned int domain; /**< \brief Domain number (xxxx in the PCI BDF notation xxxx:yy:zz.t). + * 32bits PCI domain support break the library ABI, hence it's disabled by default. */ #endif - unsigned char bus, dev, func; - unsigned short class_id; - unsigned short vendor_id, device_id, subvendor_id, subdevice_id; - unsigned char revision; - float linkspeed; /* in GB/s */ + unsigned char bus; /**< \brief Bus number (yy in the PCI BDF notation xxxx:yy:zz.t). */ + unsigned char dev; /**< \brief Device number (zz in the PCI BDF notation xxxx:yy:zz.t). */ + unsigned char func; /**< \brief Function number (t in the PCI BDF notation xxxx:yy:zz.t). */ + unsigned short class_id; /**< \brief The class number (first two bytes, without the prog_if). */ + unsigned short vendor_id; /**< \brief Vendor ID (xxxx in [xxxx:yyyy]). */ + unsigned short device_id; /**< \brief Device ID (yyyy in [xxxx:yyyy]). */ + unsigned short subvendor_id; /**< \brief Sub-Vendor ID. */ + unsigned short subdevice_id; /**< \brief Sub-Device ID. */ + unsigned char revision; /**< \brief Revision number. */ + float linkspeed; /**< \brief Link speed in GB/s. + * This datarate is the currently configured speed of the entire PCI link + * (sum of the bandwidth of all PCI lanes in that link). + * It may change during execution since some devices are able to + * slow their PCI links down when idle. + */ } pcidev; /** \brief Bridge specific Object Attributes */ struct hwloc_bridge_attr_s { union { - struct hwloc_pcidev_attr_s pci; + struct hwloc_pcidev_attr_s pci; /**< \brief PCI attribute of the upstream part as a PCI device. */ } upstream; - hwloc_obj_bridge_type_t upstream_type; + hwloc_obj_bridge_type_t upstream_type; /**< \brief Upstream Bridge type. */ union { struct { #ifndef HWLOC_HAVE_32BITS_PCI_DOMAIN - unsigned short domain; /* Only 16bits PCI domains are supported by default */ + unsigned short domain; /**< \brief Domain number the downstream PCI buses. + * Only 16bits PCI domains are supported by default. */ #else - unsigned int domain; /* 32bits PCI domain support break the library ABI, hence it's disabled by default */ + unsigned int domain; /**< \brief Domain number the downstream PCI buses. + * 32bits PCI domain support break the library ABI, hence it's disabled by default */ #endif - unsigned char secondary_bus, subordinate_bus; + unsigned char secondary_bus; /**< \brief First PCI bus number below the bridge. */ + unsigned char subordinate_bus; /**< \brief Highest PCI bus number below the bridge. */ } pci; } downstream; - hwloc_obj_bridge_type_t downstream_type; + hwloc_obj_bridge_type_t downstream_type; /**< \brief Downstream Bridge type. */ unsigned depth; } bridge; /** \brief OS Device specific Object Attributes */ @@ -667,7 +718,7 @@ union hwloc_obj_attr_u { } osdev; }; -/** \brief Object info +/** \brief Object info attribute (name and value strings) * * \sa hwlocality_info_attr */ @@ -734,6 +785,8 @@ HWLOC_DECLSPEC void hwloc_topology_destroy (hwloc_topology_t topology); * * This is useful for keeping a backup while modifying a topology. * + * \return 0 on success, -1 on error. + * * \note Object userdata is not duplicated since hwloc does not know what it point to. * The objects of both old and new topologies will point to the same userdata. */ @@ -788,6 +841,8 @@ HWLOC_DECLSPEC void hwloc_topology_check(hwloc_topology_t topology); * * This is the depth of ::HWLOC_OBJ_PU objects plus one. * + * \return the depth of the object tree. + * * \note NUMA nodes, I/O and Misc objects are ignored when computing * the depth of the tree (they are placed on special levels). */ @@ -795,23 +850,26 @@ HWLOC_DECLSPEC int hwloc_topology_get_depth(hwloc_topology_t __hwloc_restrict to /** \brief Returns the depth of objects of type \p type. * - * If no object of this type is present on the underlying architecture, or if - * the OS doesn't provide this kind of information, the function returns - * ::HWLOC_TYPE_DEPTH_UNKNOWN. + * \return The depth of objects of type \p type. * - * If type is absent but a similar type is acceptable, see also - * hwloc_get_type_or_below_depth() and hwloc_get_type_or_above_depth(). - * - * If ::HWLOC_OBJ_GROUP is given, the function may return ::HWLOC_TYPE_DEPTH_MULTIPLE - * if multiple levels of Groups exist. - * - * If a NUMA node, I/O or Misc object type is given, the function returns a virtual - * value because these objects are stored in special levels that are not CPU-related. + * \return A negative virtual depth if a NUMA node, I/O or Misc object type is given. + * These objects are stored in special levels that are not CPU-related. * This virtual depth may be passed to other hwloc functions such as * hwloc_get_obj_by_depth() but it should not be considered as an actual * depth by the application. In particular, it should not be compared with * any other object depth or with the entire topology depth. - * \sa hwloc_get_memory_parents_depth(). + * + * \return ::HWLOC_TYPE_DEPTH_UNKNOWN + * if no object of this type is present on the underlying architecture, + * or if the OS doesn't provide this kind of information. + * + * \return ::HWLOC_TYPE_DEPTH_MULTIPLE if type ::HWLOC_OBJ_GROUP is given + * and multiple levels of Groups exist. + * + * \note If the type is absent but a similar type is acceptable, see also + * hwloc_get_type_or_below_depth() and hwloc_get_type_or_above_depth(). + * + * \sa hwloc_get_memory_parents_depth() for managing the depth of memory objects. * * \sa hwloc_type_sscanf_as_depth() for returning the depth of objects * whose type is given as a string. @@ -887,18 +945,23 @@ hwloc_get_type_or_above_depth (hwloc_topology_t topology, hwloc_obj_type_t type) * \p depth should between 0 and hwloc_topology_get_depth()-1, * or a virtual depth such as ::HWLOC_TYPE_DEPTH_NUMANODE. * + * \return The type of objects at depth \p depth. * \return (hwloc_obj_type_t)-1 if depth \p depth does not exist. */ HWLOC_DECLSPEC hwloc_obj_type_t hwloc_get_depth_type (hwloc_topology_t topology, int depth) __hwloc_attribute_pure; /** \brief Returns the width of level at depth \p depth. + * + * \return The number of objects at topology depth \p depth. + * \return 0 if there are no objects at depth \p depth. */ HWLOC_DECLSPEC unsigned hwloc_get_nbobjs_by_depth (hwloc_topology_t topology, int depth) __hwloc_attribute_pure; /** \brief Returns the width of level type \p type * - * If no object for that type exists, 0 is returned. - * If there are several levels with objects of that type, -1 is returned. + * \return The number of objects of type \p type. + * \return -1 if there are multiple levels with objects of that type, e.g. ::HWLOC_OBJ_GROUP. + * \return 0 if there are no objects at depth \p depth. */ static __hwloc_inline int hwloc_get_nbobjs_by_type (hwloc_topology_t topology, hwloc_obj_type_t type) __hwloc_attribute_pure; @@ -906,34 +969,45 @@ hwloc_get_nbobjs_by_type (hwloc_topology_t topology, hwloc_obj_type_t type) __hw /** \brief Returns the top-object of the topology-tree. * * Its type is ::HWLOC_OBJ_MACHINE. + * + * This function cannot return \c NULL. */ static __hwloc_inline hwloc_obj_t hwloc_get_root_obj (hwloc_topology_t topology) __hwloc_attribute_pure; -/** \brief Returns the topology object at logical index \p idx from depth \p depth */ +/** \brief Returns the topology object at logical index \p idx from depth \p depth + * + * \return The object if it exists. + * \return \c NULL if there is no object with this index and depth. + */ HWLOC_DECLSPEC hwloc_obj_t hwloc_get_obj_by_depth (hwloc_topology_t topology, int depth, unsigned idx) __hwloc_attribute_pure; /** \brief Returns the topology object at logical index \p idx with type \p type * - * If no object for that type exists, \c NULL is returned. - * If there are several levels with objects of that type (::HWLOC_OBJ_GROUP), - * \c NULL is returned and the caller may fallback to hwloc_get_obj_by_depth(). + * \return The object if it exists. + * \return \c NULL if there is no object with this index and type. + * \return \c NULL if there are multiple levels with objects of that type (e.g. ::HWLOC_OBJ_GROUP), + * the caller may fallback to hwloc_get_obj_by_depth(). */ static __hwloc_inline hwloc_obj_t hwloc_get_obj_by_type (hwloc_topology_t topology, hwloc_obj_type_t type, unsigned idx) __hwloc_attribute_pure; /** \brief Returns the next object at depth \p depth. * - * If \p prev is \c NULL, return the first object at depth \p depth. + * \return The first object at depth \p depth if \p prev is \c NULL. + * \return The object after \p prev at depth \p depth if \p prev is not \c NULL. + * \return \c NULL if there is no such object. */ static __hwloc_inline hwloc_obj_t hwloc_get_next_obj_by_depth (hwloc_topology_t topology, int depth, hwloc_obj_t prev); /** \brief Returns the next object of type \p type. * - * If \p prev is \c NULL, return the first object at type \p type. If - * there are multiple or no depth for given type, return \c NULL and - * let the caller fallback to hwloc_get_next_obj_by_depth(). + * \return The first object of type \p type if \p prev is \c NULL. + * \return The object after \p prev of type \p type if \p prev is not \c NULL. + * \return \c NULL if there is no such object. + * \return \c NULL if there are multiple levels with objects of that type (e.g. ::HWLOC_OBJ_GROUP), + * the caller may fallback to hwloc_get_obj_by_depth(). */ static __hwloc_inline hwloc_obj_t hwloc_get_next_obj_by_type (hwloc_topology_t topology, hwloc_obj_type_t type, @@ -954,6 +1028,8 @@ hwloc_get_next_obj_by_type (hwloc_topology_t topology, hwloc_obj_type_t type, * * hwloc_obj_type_snprintf() may return a more precise output for a specific * object, but it requires the caller to provide the output buffer. + * + * \return A constant string containing the object type name or \c "Unknown". */ HWLOC_DECLSPEC const char * hwloc_obj_type_string (hwloc_obj_type_t type) __hwloc_attribute_const; @@ -972,7 +1048,7 @@ HWLOC_DECLSPEC const char * hwloc_obj_type_string (hwloc_obj_type_t type) __hwlo * If \p size is 0, \p string may safely be \c NULL. * * \return the number of characters that were actually written if not truncating, - * or that would have been written (not including the ending \\0). + * or that would have been written (not including the ending \c \0). */ HWLOC_DECLSPEC int hwloc_obj_type_snprintf(char * __hwloc_restrict string, size_t size, hwloc_obj_t obj, @@ -987,7 +1063,7 @@ HWLOC_DECLSPEC int hwloc_obj_type_snprintf(char * __hwloc_restrict string, size_ * If \p size is 0, \p string may safely be \c NULL. * * \return the number of characters that were actually written if not truncating, - * or that would have been written (not including the ending \\0). + * or that would have been written (not including the ending \c \0). */ HWLOC_DECLSPEC int hwloc_obj_attr_snprintf(char * __hwloc_restrict string, size_t size, hwloc_obj_t obj, const char * __hwloc_restrict separator, @@ -1049,23 +1125,26 @@ HWLOC_DECLSPEC int hwloc_type_sscanf_as_depth(const char *string, -/** \defgroup hwlocality_info_attr Consulting and Adding Key-Value Info Attributes +/** \defgroup hwlocality_info_attr Consulting and Adding Info Attributes * * @{ */ -/** \brief Search the given key name in object infos and return the corresponding value. +/** \brief Search the given name in object infos and return the corresponding value. * - * If multiple keys match the given name, only the first one is returned. + * If multiple info attributes match the given name, only the first one is returned. * - * \return \c NULL if no such key exists. + * \return A pointer to the value string if it exists. + * \return \c NULL if no such info attribute exists. + * + * \note The string should not be freed by the caller, it belongs to the hwloc library. */ static __hwloc_inline const char * hwloc_obj_get_info_by_name(hwloc_obj_t obj, const char *name) __hwloc_attribute_pure; -/** \brief Add the given info name and value pair to the given object. +/** \brief Add the given name and value pair to the given object info attributes. * - * The info is appended to the existing info array even if another key + * The info pair is appended to the existing info array even if another pair * with the same name already exists. * * The input strings are copied before being added in the object infos. @@ -1073,14 +1152,30 @@ hwloc_obj_get_info_by_name(hwloc_obj_t obj, const char *name) __hwloc_attribute_ * \return \c 0 on success, \c -1 on error. * * \note This function may be used to enforce object colors in the lstopo - * graphical output by using "lstopoStyle" as a name and "Background=#rrggbb" + * graphical output by adding "lstopoStyle" as a name and "Background=#rrggbb" * as a value. See CUSTOM COLORS in the lstopo(1) manpage for details. * - * \note If \p value contains some non-printable characters, they will + * \note If \p name or \p value contain some non-printable characters, they will * be dropped when exporting to XML, see hwloc_topology_export_xml() in hwloc/export.h. */ HWLOC_DECLSPEC int hwloc_obj_add_info(hwloc_obj_t obj, const char *name, const char *value); +/** \brief Set (or replace) the subtype of an object. + * + * The given \p subtype is copied internally, the caller is responsible + * for freeing the original \p subtype if needed. + * + * If another subtype already exists in \p object, it is replaced. + * The given \p subtype may be \c NULL to remove the existing subtype. + * + * \note This function is mostly meant to initialize the subtype of user-added + * objects such as groups with hwloc_topology_alloc_group_object(). + * + * \return \c 0 on success. + * \return \c -1 with \p errno set to \c ENOMEM on failure to allocate memory. + */ +HWLOC_DECLSPEC int hwloc_obj_set_subtype(hwloc_topology_t topology, hwloc_obj_t obj, const char *subtype); + /** @} */ @@ -1193,7 +1288,7 @@ typedef enum { * a problem for the application, but if it is, setting this flag * will make hwloc avoid using OS functions that would also bind * memory. This will however reduce the support of CPU bindings, - * i.e. potentially return -1 with errno set to ENOSYS in some + * i.e. potentially return -1 with errno set to \c ENOSYS in some * cases. * * This flag is only meaningful when used with functions that set @@ -1206,8 +1301,9 @@ typedef enum { /** \brief Bind current process or thread on CPUs given in physical bitmap \p set. * - * \return -1 with errno set to ENOSYS if the action is not supported - * \return -1 with errno set to EXDEV if the binding cannot be enforced + * \return 0 on success. + * \return -1 with errno set to \c ENOSYS if the action is not supported. + * \return -1 with errno set to \c EXDEV if the binding cannot be enforced. */ HWLOC_DECLSPEC int hwloc_set_cpubind(hwloc_topology_t topology, hwloc_const_cpuset_t set, int flags); @@ -1216,10 +1312,14 @@ HWLOC_DECLSPEC int hwloc_set_cpubind(hwloc_topology_t topology, hwloc_const_cpus * The CPU-set \p set (previously allocated by the caller) * is filled with the list of PUs which the process or * thread (according to \e flags) was last bound to. + * + * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_get_cpubind(hwloc_topology_t topology, hwloc_cpuset_t set, int flags); /** \brief Bind a process \p pid on CPUs given in physical bitmap \p set. + * + * \return 0 on success, -1 on error. * * \note \p hwloc_pid_t is \p pid_t on Unix platforms, * and \p HANDLE on native Windows platforms. @@ -1238,6 +1338,8 @@ HWLOC_DECLSPEC int hwloc_set_proc_cpubind(hwloc_topology_t topology, hwloc_pid_t * is filled with the list of PUs which the process * was last bound to. * + * \return 0 on success, -1 on error. + * * \note \p hwloc_pid_t is \p pid_t on Unix platforms, * and \p HANDLE on native Windows platforms. * @@ -1251,6 +1353,8 @@ HWLOC_DECLSPEC int hwloc_get_proc_cpubind(hwloc_topology_t topology, hwloc_pid_t #ifdef hwloc_thread_t /** \brief Bind a thread \p thread on CPUs given in physical bitmap \p set. + * + * \return 0 on success, -1 on error. * * \note \p hwloc_thread_t is \p pthread_t on Unix platforms, * and \p HANDLE on native Windows platforms. @@ -1267,6 +1371,8 @@ HWLOC_DECLSPEC int hwloc_set_thread_cpubind(hwloc_topology_t topology, hwloc_thr * is filled with the list of PUs which the thread * was last bound to. * + * \return 0 on success, -1 on error. + * * \note \p hwloc_thread_t is \p pthread_t on Unix platforms, * and \p HANDLE on native Windows platforms. * @@ -1291,6 +1397,8 @@ HWLOC_DECLSPEC int hwloc_get_thread_cpubind(hwloc_topology_t topology, hwloc_thr * on which all threads are running), or only the current thread. If the * process is single-threaded, flags can be set to zero to let hwloc use * whichever method is available on the underlying OS. + * + * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_get_last_cpu_location(hwloc_topology_t topology, hwloc_cpuset_t set, int flags); @@ -1305,6 +1413,8 @@ HWLOC_DECLSPEC int hwloc_get_last_cpu_location(hwloc_topology_t topology, hwloc_ * so this function may return something that is already * outdated. * + * \return 0 on success, -1 on error. + * * \note \p hwloc_pid_t is \p pid_t on Unix platforms, * and \p HANDLE on native Windows platforms. * @@ -1343,7 +1453,7 @@ HWLOC_DECLSPEC int hwloc_get_proc_last_cpu_location(hwloc_topology_t topology, h * (e.g., some systems only allow binding memory on a per-thread * basis, whereas other systems only allow binding memory for all * threads in a process). - * \p errno will be set to EXDEV when the requested set can not be enforced + * \p errno will be set to \c EXDEV when the requested set can not be enforced * (e.g., some systems only allow binding memory to a single NUMA node). * * If ::HWLOC_MEMBIND_STRICT was not passed, the function may fail as well, @@ -1417,6 +1527,12 @@ typedef enum { HWLOC_MEMBIND_FIRSTTOUCH = 1, /** \brief Allocate memory on the specified nodes. + * + * The actual behavior may slightly vary between operating systems, + * especially when (some of) the requested nodes are full. + * On Linux, by default, the MPOL_PREFERRED_MANY (or MPOL_PREFERRED) policy + * is used. However, if the hwloc strict flag is also given, the Linux + * MPOL_BIND policy is rather used. * \hideinitializer */ HWLOC_MEMBIND_BIND = 2, @@ -1429,6 +1545,16 @@ typedef enum { * \hideinitializer */ HWLOC_MEMBIND_INTERLEAVE = 3, + /** \brief Allocate memory on the given nodes in an interleaved + * / weighted manner. The precise layout of the memory across + * multiple NUMA nodes is OS/system specific. Weighted interleaving + * can be useful when threads distributed across the specified NUMA + * nodes with different bandwidth capabilities will all be accessing + * the whole memory range concurrently, since the interleave will then + * balance the memory references. + * \hideinitializer */ + HWLOC_MEMBIND_WEIGHTED_INTERLEAVE = 5, + /** \brief For each page bound with this policy, by next time * it is touched (and next time only), it is moved from its current * location to the local NUMA node of the thread where the memory @@ -1492,7 +1618,7 @@ typedef enum { * could potentially affect CPU bindings. Note, however, that using * NOCPUBIND may reduce hwloc's overall memory binding * support. Specifically: some of hwloc's memory binding functions - * may fail with errno set to ENOSYS when used with NOCPUBIND. + * may fail with errno set to \c ENOSYS when used with NOCPUBIND. * \hideinitializer */ HWLOC_MEMBIND_NOCPUBIND = (1<<4), @@ -1521,8 +1647,9 @@ typedef enum { * If ::HWLOC_MEMBIND_BYNODESET is specified, set is considered a nodeset. * Otherwise it's a cpuset. * - * \return -1 with errno set to ENOSYS if the action is not supported - * \return -1 with errno set to EXDEV if the binding cannot be enforced + * \return 0 on success. + * \return -1 with errno set to \c ENOSYS if the action is not supported. + * \return -1 with errno set to \c EXDEV if the binding cannot be enforced. */ HWLOC_DECLSPEC int hwloc_set_membind(hwloc_topology_t topology, hwloc_const_bitmap_t set, hwloc_membind_policy_t policy, int flags); @@ -1551,7 +1678,7 @@ HWLOC_DECLSPEC int hwloc_set_membind(hwloc_topology_t topology, hwloc_const_bitm * ::HWLOC_MEMBIND_STRICT is only meaningful when ::HWLOC_MEMBIND_PROCESS * is also specified. In this case, hwloc will check the default * memory policies and nodesets for all threads in the process. If - * they are not identical, -1 is returned and errno is set to EXDEV. + * they are not identical, -1 is returned and errno is set to \c EXDEV. * If they are identical, the values are returned in \p set and \p * policy. * @@ -1571,7 +1698,9 @@ HWLOC_DECLSPEC int hwloc_set_membind(hwloc_topology_t topology, hwloc_const_bitm * Otherwise it's a cpuset. * * If any other flags are specified, -1 is returned and errno is set - * to EINVAL. + * to \c EINVAL. + * + * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_get_membind(hwloc_topology_t topology, hwloc_bitmap_t set, hwloc_membind_policy_t * policy, int flags); @@ -1581,8 +1710,9 @@ HWLOC_DECLSPEC int hwloc_get_membind(hwloc_topology_t topology, hwloc_bitmap_t s * If ::HWLOC_MEMBIND_BYNODESET is specified, set is considered a nodeset. * Otherwise it's a cpuset. * - * \return -1 with errno set to ENOSYS if the action is not supported - * \return -1 with errno set to EXDEV if the binding cannot be enforced + * \return 0 on success. + * \return -1 with errno set to \c ENOSYS if the action is not supported. + * \return -1 with errno set to \c EXDEV if the binding cannot be enforced. * * \note \p hwloc_pid_t is \p pid_t on Unix platforms, * and \p HANDLE on native Windows platforms. @@ -1614,7 +1744,7 @@ HWLOC_DECLSPEC int hwloc_set_proc_membind(hwloc_topology_t topology, hwloc_pid_t * If ::HWLOC_MEMBIND_STRICT is specified, hwloc will check the default * memory policies and nodesets for all threads in the specified * process. If they are not identical, -1 is returned and errno is - * set to EXDEV. If they are identical, the values are returned in \p + * set to \c EXDEV. If they are identical, the values are returned in \p * set and \p policy. * * Otherwise, \p set is set to the logical OR of all threads' @@ -1626,7 +1756,9 @@ HWLOC_DECLSPEC int hwloc_set_proc_membind(hwloc_topology_t topology, hwloc_pid_t * Otherwise it's a cpuset. * * If any other flags are specified, -1 is returned and errno is set - * to EINVAL. + * to \c EINVAL. + * + * \return 0 on success, -1 on error. * * \note \p hwloc_pid_t is \p pid_t on Unix platforms, * and \p HANDLE on native Windows platforms. @@ -1639,9 +1771,9 @@ HWLOC_DECLSPEC int hwloc_get_proc_membind(hwloc_topology_t topology, hwloc_pid_t * If ::HWLOC_MEMBIND_BYNODESET is specified, set is considered a nodeset. * Otherwise it's a cpuset. * - * \return 0 if \p len is 0. - * \return -1 with errno set to ENOSYS if the action is not supported - * \return -1 with errno set to EXDEV if the binding cannot be enforced + * \return 0 on success or if \p len is 0. + * \return -1 with errno set to \c ENOSYS if the action is not supported. + * \return -1 with errno set to \c EXDEV if the binding cannot be enforced. */ HWLOC_DECLSPEC int hwloc_set_area_membind(hwloc_topology_t topology, const void *addr, size_t len, hwloc_const_bitmap_t set, hwloc_membind_policy_t policy, int flags); @@ -1658,7 +1790,7 @@ HWLOC_DECLSPEC int hwloc_set_area_membind(hwloc_topology_t topology, const void * * If ::HWLOC_MEMBIND_STRICT is specified, the target pages are first * checked to see if they all have the same memory binding policy and - * nodeset. If they do not, -1 is returned and errno is set to EXDEV. + * nodeset. If they do not, -1 is returned and errno is set to \c EXDEV. * If they are identical across all pages, the set and policy are * returned in \p set and \p policy, respectively. * @@ -1671,9 +1803,10 @@ HWLOC_DECLSPEC int hwloc_set_area_membind(hwloc_topology_t topology, const void * Otherwise it's a cpuset. * * If any other flags are specified, -1 is returned and errno is set - * to EINVAL. + * to \c EINVAL. * - * If \p len is 0, -1 is returned and errno is set to EINVAL. + * \return 0 on success. + * \return -1 with errno set to \c EINVAL if \p len is 0. */ HWLOC_DECLSPEC int hwloc_get_area_membind(hwloc_topology_t topology, const void *addr, size_t len, hwloc_bitmap_t set, hwloc_membind_policy_t * policy, int flags); @@ -1696,6 +1829,8 @@ HWLOC_DECLSPEC int hwloc_get_area_membind(hwloc_topology_t topology, const void * considered a nodeset. Otherwise it's a cpuset. * * If \p len is 0, \p set is emptied. + * + * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_get_area_memlocation(hwloc_topology_t topology, const void *addr, size_t len, hwloc_bitmap_t set, int flags); @@ -1704,17 +1839,20 @@ HWLOC_DECLSPEC int hwloc_get_area_memlocation(hwloc_topology_t topology, const v * This is equivalent to malloc(), except that it tries to allocate * page-aligned memory from the OS. * + * \return a pointer to the allocated area, or \c NULL on error. + * * \note The allocated memory should be freed with hwloc_free(). */ HWLOC_DECLSPEC void *hwloc_alloc(hwloc_topology_t topology, size_t len); /** \brief Allocate some memory on NUMA memory nodes specified by \p set * - * \return NULL with errno set to ENOSYS if the action is not supported - * and ::HWLOC_MEMBIND_STRICT is given - * \return NULL with errno set to EXDEV if the binding cannot be enforced - * and ::HWLOC_MEMBIND_STRICT is given - * \return NULL with errno set to ENOMEM if the memory allocation failed + * \return a pointer to the allocated area. + * \return NULL with errno set to \c ENOSYS if the action is not supported + * and ::HWLOC_MEMBIND_STRICT is given. + * \return NULL with errno set to \c EXDEV if the binding cannot be enforced + * and ::HWLOC_MEMBIND_STRICT is given. + * \return NULL with errno set to \c ENOMEM if the memory allocation failed * even before trying to bind. * * If ::HWLOC_MEMBIND_BYNODESET is specified, set is considered a nodeset. @@ -1735,12 +1873,16 @@ HWLOC_DECLSPEC void *hwloc_alloc_membind(hwloc_topology_t topology, size_t len, * * If ::HWLOC_MEMBIND_BYNODESET is specified, set is considered a nodeset. * Otherwise it's a cpuset. + * + * \return a pointer to the allocated area, or \c NULL on error. */ static __hwloc_inline void * hwloc_alloc_membind_policy(hwloc_topology_t topology, size_t len, hwloc_const_bitmap_t set, hwloc_membind_policy_t policy, int flags) __hwloc_attribute_malloc; /** \brief Free memory that was previously allocated by hwloc_alloc() * or hwloc_alloc_membind(). + * + * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_free(hwloc_topology_t topology, void *addr, size_t len); @@ -1749,6 +1891,9 @@ HWLOC_DECLSPEC int hwloc_free(hwloc_topology_t topology, void *addr, size_t len) /** \defgroup hwlocality_setsource Changing the Source of Topology Discovery + * + * These functions must be called between hwloc_topology_init() and hwloc_topology_load(). + * Otherwise, they will return -1 with errno set to \c EBUSY. * * If none of the functions below is called, the default is to detect all the objects * of the machine that the caller is allowed to access. @@ -1777,8 +1922,14 @@ HWLOC_DECLSPEC int hwloc_free(hwloc_topology_t topology, void *addr, size_t len) * \note \p hwloc_pid_t is \p pid_t on Unix platforms, * and \p HANDLE on native Windows platforms. * - * \note -1 is returned and errno is set to ENOSYS on platforms that do not + * \note -1 is returned and errno is set to \c ENOSYS on platforms that do not * support this feature. + * + * \note The PID will not actually be used until hwloc_topology_load(). + * If the corresponding process exits in the meantime, hwloc will ignore the PID. + * If another process reuses the PID, the view of that process will be used. + * + * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_topology_set_pid(hwloc_topology_t __hwloc_restrict topology, hwloc_pid_t pid); @@ -1796,13 +1947,16 @@ HWLOC_DECLSPEC int hwloc_topology_set_pid(hwloc_topology_t __hwloc_restrict topo * * If \p description was properly parsed and describes a valid topology * configuration, this function returns 0. - * Otherwise -1 is returned and errno is set to EINVAL. + * Otherwise -1 is returned and errno is set to \c EINVAL. * * Note that this function does not actually load topology * information; it just tells hwloc where to load it from. You'll * still need to invoke hwloc_topology_load() to actually load the * topology information. * + * \return 0 on success. + * \return -1 with errno set to \c EINVAL if the description was invalid. + * * \note For convenience, this backend provides empty binding hooks which just * return success. * @@ -1824,7 +1978,8 @@ HWLOC_DECLSPEC int hwloc_topology_set_synthetic(hwloc_topology_t __hwloc_restric * still need to invoke hwloc_topology_load() to actually load the * topology information. * - * \return -1 with errno set to EINVAL on failure to read the XML file. + * \return 0 on success. + * \return -1 with errno set to \c EINVAL on failure to read the XML file. * * \note See also hwloc_topology_set_userdata_import_callback() * for importing application-specific object userdata. @@ -1837,22 +1992,28 @@ HWLOC_DECLSPEC int hwloc_topology_set_synthetic(hwloc_topology_t __hwloc_restric * \note On success, the XML component replaces the previously enabled * component (if any), but the topology is not actually modified until * hwloc_topology_load(). + * + * \note If an invalid XML input file is given, the error may be reported + * either here or later by hwloc_topology_load() depending on the XML library + * used by hwloc. */ HWLOC_DECLSPEC int hwloc_topology_set_xml(hwloc_topology_t __hwloc_restrict topology, const char * __hwloc_restrict xmlpath); /** \brief Enable XML based topology using a memory buffer (instead of * a file, as with hwloc_topology_set_xml()). * - * Gather topology information from the XML memory buffer given at \p - * buffer and of length \p size. This buffer may have been filled - * earlier with hwloc_topology_export_xmlbuffer() in hwloc/export.h. + * Gather topology information from the XML memory buffer given at + * \p buffer and of length \p size (including an ending \c \0). + * This buffer may have been filled earlier with + * hwloc_topology_export_xmlbuffer() in hwloc/export.h. * * Note that this function does not actually load topology * information; it just tells hwloc where to load it from. You'll * still need to invoke hwloc_topology_load() to actually load the * topology information. * - * \return -1 with errno set to EINVAL on failure to read the XML buffer. + * \return 0 on success. + * \return -1 with errno set to \c EINVAL on failure to read the XML buffer. * * \note See also hwloc_topology_set_userdata_import_callback() * for importing application-specific object userdata. @@ -1865,6 +2026,10 @@ HWLOC_DECLSPEC int hwloc_topology_set_xml(hwloc_topology_t __hwloc_restrict topo * \note On success, the XML component replaces the previously enabled * component (if any), but the topology is not actually modified until * hwloc_topology_load(). + * + * \note If an invalid XML input file is given, the error may be reported + * either here or later by hwloc_topology_load() depending on the XML library + * used by hwloc. */ HWLOC_DECLSPEC int hwloc_topology_set_xmlbuffer(hwloc_topology_t __hwloc_restrict topology, const char * __hwloc_restrict buffer, int size); @@ -1890,6 +2055,9 @@ enum hwloc_topology_components_flag_e { * This may be used to avoid expensive parts of the discovery process. * For instance, CUDA-specific discovery may be expensive and unneeded * while generic I/O discovery could still be useful. + * + * \return 0 on success. + * \return -1 on error, for instance if flags are invalid. */ HWLOC_DECLSPEC int hwloc_topology_set_components(hwloc_topology_t __hwloc_restrict topology, unsigned long flags, const char * __hwloc_restrict name); @@ -2069,9 +2237,10 @@ enum hwloc_topology_flags_e { */ HWLOC_TOPOLOGY_FLAG_NO_DISTANCES = (1UL<<7), - /** \brief Ignore memory attributes. + /** \brief Ignore memory attributes and tiers. * - * Ignore memory attribues from the operating systems (and from XML). + * Ignore memory attribues from the operating systems (and from XML) + * Hence also do not try to build memory tiers. */ HWLOC_TOPOLOGY_FLAG_NO_MEMATTRS = (1UL<<8), @@ -2092,6 +2261,9 @@ enum hwloc_topology_flags_e { * By default, no flags are set (\c 0). * * The flags set in a topology may be retrieved with hwloc_topology_get_flags(). + * + * \return 0 on success. + * \return -1 on error, for instance if flags are invalid. */ HWLOC_DECLSPEC int hwloc_topology_set_flags (hwloc_topology_t topology, unsigned long flags); @@ -2103,6 +2275,8 @@ HWLOC_DECLSPEC int hwloc_topology_set_flags (hwloc_topology_t topology, unsigned * no flags are set (\c 0 is returned). * * \return the flags previously set with hwloc_topology_set_flags(). + * + * \note This function may also be called after hwloc_topology_load(). */ HWLOC_DECLSPEC unsigned long hwloc_topology_get_flags (hwloc_topology_t topology); @@ -2112,6 +2286,8 @@ HWLOC_DECLSPEC unsigned long hwloc_topology_get_flags (hwloc_topology_t topology * running this program. * \return 0 instead (for instance if using another file-system root, * a XML topology file, or a synthetic topology). + * + * \note This function may also be called after hwloc_topology_load(). */ HWLOC_DECLSPEC int hwloc_topology_is_thissystem(hwloc_topology_t __hwloc_restrict topology) __hwloc_attribute_pure; @@ -2197,6 +2373,8 @@ struct hwloc_topology_membind_support { unsigned char migrate_membind; /** Getting the last NUMA nodes where a memory area was allocated is supported */ unsigned char get_area_memlocation; + /** Weighted interleave policy is supported. */ + unsigned char weighted_interleave_membind; }; /** \brief Flags describing miscellaneous features. @@ -2239,14 +2417,22 @@ struct hwloc_topology_support { * to report the supported features of the original remote machine * instead. If it was successfully imported, \p imported_support * will be set in the struct hwloc_topology_misc_support array. + * + * \return A pointer to a support structure. + * + * \note The function cannot return \c NULL. + * \note The returned pointer should not be freed, it belongs to the hwloc library. + * + * \note This function may be called before or after hwloc_topology_load() + * but the support structure only contains valid information after. */ HWLOC_DECLSPEC const struct hwloc_topology_support *hwloc_topology_get_support(hwloc_topology_t __hwloc_restrict topology); /** \brief Type filtering flags. * * By default, most objects are kept (::HWLOC_TYPE_FILTER_KEEP_ALL). - * Instruction caches, I/O and Misc objects are ignored by default (::HWLOC_TYPE_FILTER_KEEP_NONE). - * Die and Group levels are ignored unless they bring structure (::HWLOC_TYPE_FILTER_KEEP_STRUCTURE). + * Instruction caches, memory-side caches, I/O and Misc objects are ignored by default (::HWLOC_TYPE_FILTER_KEEP_NONE). + * Group levels are ignored unless they bring structure (::HWLOC_TYPE_FILTER_KEEP_STRUCTURE). * * Note that group objects are also ignored individually (without the entire level) * when they do not bring structure. @@ -2298,32 +2484,44 @@ enum hwloc_type_filter_e { }; /** \brief Set the filtering for the given object type. + * + * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_topology_set_type_filter(hwloc_topology_t topology, hwloc_obj_type_t type, enum hwloc_type_filter_e filter); /** \brief Get the current filtering for the given object type. + * + * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_topology_get_type_filter(hwloc_topology_t topology, hwloc_obj_type_t type, enum hwloc_type_filter_e *filter); /** \brief Set the filtering for all object types. * * If some types do not support this filtering, they are silently ignored. + * + * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_topology_set_all_types_filter(hwloc_topology_t topology, enum hwloc_type_filter_e filter); /** \brief Set the filtering for all CPU cache object types. * * Memory-side caches are not involved since they are not CPU caches. + * + * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_topology_set_cache_types_filter(hwloc_topology_t topology, enum hwloc_type_filter_e filter); /** \brief Set the filtering for all CPU instruction cache object types. * * Memory-side caches are not involved since they are not CPU caches. + * + * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_topology_set_icache_types_filter(hwloc_topology_t topology, enum hwloc_type_filter_e filter); /** \brief Set the filtering for all I/O object types. + * + * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_topology_set_io_types_filter(hwloc_topology_t topology, enum hwloc_type_filter_e filter); @@ -2343,6 +2541,9 @@ HWLOC_DECLSPEC void hwloc_topology_set_userdata(hwloc_topology_t topology, const * * Retrieve the application-given private data pointer that was * previously set with hwloc_topology_set_userdata(). + * + * \return A pointer to the private-data if any. + * \return \c NULL if no private-data was previoulsy set. */ HWLOC_DECLSPEC void * hwloc_topology_get_userdata(hwloc_topology_t topology); @@ -2395,21 +2596,32 @@ enum hwloc_restrict_flags_e { * are not included (or partially included) in the CPU set \p set. * All objects CPU and node sets are restricted accordingly. * + * By default, \p set is a CPU set. It means that the set of PUs in + * the topology is restricted. Once some PUs got removed, their parents + * may also get removed recursively if they became child-less. + * * If ::HWLOC_RESTRICT_FLAG_BYNODESET is passed in \p flags, * \p set is considered a nodeset instead of a CPU set. + * It means that the set of NUMA nodes in the topology is restricted + * (instead of PUs). Once some NUMA nodes got removed, their parents + * may also get removed recursively if they became child-less. * * \p flags is a OR'ed set of ::hwloc_restrict_flags_e. * + * \note Restricting the topology removes some locality information, + * hence the remaining objects may get reordered (including PUs and NUMA nodes), + * and their logical indexes may change. + * * \note This call may not be reverted by restricting back to a larger * set. Once dropped during restriction, objects may not be brought * back, except by loading another topology with hwloc_topology_load(). * * \return 0 on success. * - * \return -1 with errno set to EINVAL if the input set is invalid. + * \return -1 with errno set to \c EINVAL if the input set is invalid. * The topology is not modified in this case. * - * \return -1 with errno set to ENOMEM on failure to allocate internal data. + * \return -1 with errno set to \c ENOMEM on failure to allocate internal data. * The topology is reinitialized in this case. It should be either * destroyed with hwloc_topology_destroy() or configured and loaded again. */ @@ -2449,6 +2661,8 @@ enum hwloc_allow_flags_e { * * \p flags must be set to one flag among ::hwloc_allow_flags_e. * + * \return 0 on success, -1 on error. + * * \note Removing objects from a topology should rather be performed with * hwloc_topology_restrict(). */ @@ -2466,6 +2680,9 @@ HWLOC_DECLSPEC int hwloc_topology_allow(hwloc_topology_t __hwloc_restrict topolo * * The new leaf object will not have any \p cpuset. * + * The \p subtype object attribute may be defined with hwloc_obj_set_subtype() + * after successful insertion. + * * \return the newly-created object * * \return \c NULL on error. @@ -2482,10 +2699,33 @@ HWLOC_DECLSPEC hwloc_obj_t hwloc_topology_insert_misc_object(hwloc_topology_t to * This function returns a new Group object. * * The caller should (at least) initialize its sets before inserting - * the object in the topology. See hwloc_topology_insert_group_object(). - */ + * the object in the topology, see hwloc_topology_insert_group_object(). + * Or it may decide not to insert and just free the group object + * by calling hwloc_topology_free_group_object(). + * + * \return The allocated object on success. + * \return \c NULL on error. + * + * \note If successfully inserted by hwloc_topology_insert_group_object(), + * the object will be freed when the entire topology is freed. + * If insertion failed (e.g. \c NULL or empty CPU and node-sets), + * it is freed before returning the error. + */ HWLOC_DECLSPEC hwloc_obj_t hwloc_topology_alloc_group_object(hwloc_topology_t topology); +/** \brief Free a group object allocated with hwloc_topology_alloc_group_object(). + * + * This function is only useful if the group object was not given + * to hwloc_topology_insert_group_object() as planned. + * + * \note \p topology must be the same as the one previously passed + * to hwloc_topology_alloc_group_object(). + * + * \return \c 0 on success. + * \return \c -1 on error, for instance if an invalid topology is given. + */ +HWLOC_DECLSPEC int hwloc_topology_free_group_object(hwloc_topology_t topology, hwloc_obj_t group); + /** \brief Add more structure to the topology by adding an intermediate Group * * The caller should first allocate a new Group object with hwloc_topology_alloc_group_object(). @@ -2493,20 +2733,38 @@ HWLOC_DECLSPEC hwloc_obj_t hwloc_topology_alloc_group_object(hwloc_topology_t to * the final location of the Group in the topology. * Then the object can be passed to this function for actual insertion in the topology. * - * Either the cpuset or nodeset field (or both, if compatible) must be set - * to a non-empty bitmap. The complete_cpuset or complete_nodeset may be set - * instead if inserting with respect to the complete topology + * The main use case for this function is to group a subset of + * siblings among the list of children below a single parent. + * For instance, if grouping 4 cores out of a 8-core socket, + * the logical list of cores will be reordered so that the 4 grouped + * ones are consecutive. + * Then, if needed, a new depth is added between the parent and those + * children, and the Group is inserted there. + * At the end, the 4 grouped cores are now children of the Group, + * which replaces them as a child of the original parent. + * + * In practice, the grouped objects are specified through cpusets + * and/or nodesets, for instance using hwloc_obj_add_other_obj_sets() + * iteratively. + * Hence it is possible to group objects that are not children of the + * same parent, for instance some PUs below the 4 cores in example above. + * However this general case may fail if the expected Group conflicts + * with the existing hierarchy. + * For instance if each core has two PUs, it is not possible to insert + * a Group containing a single PU of each core. + * + * To specify the objects to group, either the cpuset or nodeset field + * (or both, if compatible) must be set to a non-empty bitmap. + * The complete_cpuset or complete_nodeset may be set instead if + * inserting with respect to the complete topology * (including disallowed, offline or unknown objects). - * If grouping several objects, hwloc_obj_add_other_obj_sets() is an easy way - * to build the Group sets iteratively. * These sets cannot be larger than the current topology, or they would get * restricted silently. * The core will setup the other sets after actual insertion. * - * The \p subtype object attribute may be defined (to a dynamically - * allocated string) to display something else than "Group" as the - * type name for this object in lstopo. - * Custom name/value info pairs may be added with hwloc_obj_add_info() after + * The \p subtype object attribute may be defined with hwloc_obj_set_subtype() + * to display something else than "Group" as the type name for this object in lstopo. + * Custom name-value info pairs may be added with hwloc_obj_add_info() after * insertion. * * The group \p dont_merge attribute may be set to \c 1 to prevent @@ -2519,6 +2777,18 @@ HWLOC_DECLSPEC hwloc_obj_t hwloc_topology_alloc_group_object(hwloc_topology_t to * as \c 0xffffffff to tell hwloc that this new Group should always * be discarded in favor of any existing Group with the same locality. * + * \note Inserting a group adds some locality information to the topology, + * hence the existing objects may get reordered (including PUs and NUMA nodes), + * and their logical indexes may change. + * + * \note If the insertion fails, the input group object is freed. + * + * \note If the group object should be discarded instead of inserted, + * it may be passed to hwloc_topology_free_group_object() instead. + * + * \note \p topology must be the same as the one previously passed + * to hwloc_topology_alloc_group_object(). + * * \return The inserted object if it was properly inserted. * * \return An existing object if the Group was merged or discarded @@ -2542,6 +2812,9 @@ HWLOC_DECLSPEC hwloc_obj_t hwloc_topology_insert_group_object(hwloc_topology_t t * This function is convenient between hwloc_topology_alloc_group_object() * and hwloc_topology_insert_group_object(). It builds the sets of the new Group * that will be inserted as a new intermediate parent of several objects. + * + * \return 0 on success. + * \return -1 with errno set to \c ENOMEM if some internal reallocation failed. */ HWLOC_DECLSPEC int hwloc_obj_add_other_obj_sets(hwloc_obj_t dst, hwloc_obj_t src); @@ -2558,6 +2831,9 @@ HWLOC_DECLSPEC int hwloc_obj_add_other_obj_sets(hwloc_obj_t dst, hwloc_obj_t src * attributes, etc. * * See also \ref threadsafety + * + * \return 0 on success. + * \return -1 on error, for instance if some internal reallocation failed. */ HWLOC_DECLSPEC int hwloc_topology_refresh(hwloc_topology_t topology); diff --git a/src/3rdparty/hwloc/include/hwloc/autogen/config.h b/src/3rdparty/hwloc/include/hwloc/autogen/config.h index 6f45f734..deadcf11 100644 --- a/src/3rdparty/hwloc/include/hwloc/autogen/config.h +++ b/src/3rdparty/hwloc/include/hwloc/autogen/config.h @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2023 Inria. All rights reserved. + * Copyright © 2009-2025 Inria. All rights reserved. * Copyright © 2009-2012 Université Bordeaux * Copyright © 2009-2011 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -11,10 +11,10 @@ #ifndef HWLOC_CONFIG_H #define HWLOC_CONFIG_H -#define HWLOC_VERSION "2.10.0" +#define HWLOC_VERSION "2.12.1" #define HWLOC_VERSION_MAJOR 2 -#define HWLOC_VERSION_MINOR 10 -#define HWLOC_VERSION_RELEASE 0 +#define HWLOC_VERSION_MINOR 12 +#define HWLOC_VERSION_RELEASE 1 #define HWLOC_VERSION_GREEK "" #define __hwloc_restrict diff --git a/src/3rdparty/hwloc/include/hwloc/bitmap.h b/src/3rdparty/hwloc/include/hwloc/bitmap.h index 6b56bcb9..8ce19057 100644 --- a/src/3rdparty/hwloc/include/hwloc/bitmap.h +++ b/src/3rdparty/hwloc/include/hwloc/bitmap.h @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2023 Inria. All rights reserved. + * Copyright © 2009-2024 Inria. All rights reserved. * Copyright © 2009-2012 Université Bordeaux * Copyright © 2009-2011 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -113,51 +113,88 @@ HWLOC_DECLSPEC int hwloc_bitmap_copy(hwloc_bitmap_t dst, hwloc_const_bitmap_t sr * Bitmap/String Conversion */ -/** \brief Stringify a bitmap. +/** \brief Stringify a bitmap in the default hwloc format. + * + * Note that if the bitmap is a CPU or nodeset, it contains physical indexes. + * + * Print the bits set inside a bitmap as a comma-separated list of hexadecimal 32-bit blocks. + * A bitmap containing bits 1, 33, 34, and all from 64 to 95 is printed as "0xffffffff,0x00000006,0x00000002". * * Up to \p buflen characters may be written in buffer \p buf. * * If \p buflen is 0, \p buf may safely be \c NULL. * * \return the number of characters that were actually written if not truncating, - * or that would have been written (not including the ending \\0). + * or that would have been written (not including the ending \c \0). + * \return -1 on error. */ HWLOC_DECLSPEC int hwloc_bitmap_snprintf(char * __hwloc_restrict buf, size_t buflen, hwloc_const_bitmap_t bitmap); -/** \brief Stringify a bitmap into a newly allocated string. +/** \brief Stringify a bitmap into a newly allocated string in the default hwloc format. * - * \return 0 on success, -1 on error. + * Note that if the bitmap is a CPU or nodeset, it contains physical indexes. + * + * Print the bits set inside a bitmap as a comma-separated list of hexadecimal 32-bit blocks. + * A bitmap containing bits 1, 33, 34, and all from 64 to 95 is printed as "0xffffffff,0x00000006,0x00000002". + * + * \return the number of characters that were written (not including the ending \c \0). + * \return -1 on error, for instance with \p errno set to \c ENOMEM on failure to allocate the output string. */ HWLOC_DECLSPEC int hwloc_bitmap_asprintf(char ** strp, hwloc_const_bitmap_t bitmap); -/** \brief Parse a bitmap string and stores it in bitmap \p bitmap. +/** \brief Parse a bitmap string as the default hwloc format and stores it in bitmap \p bitmap. + * + * Note that if the bitmap is a CPU or nodeset, the input string must contain physical indexes. + * + * The input string should be a comma-separared list of hexadecimal 32-bit blocks. + * String "0xffffffff,0x6,0x2" is parsed as a bitmap containing all bits between 64 and 95, + * and bits 33, 34 and 1. * * \return 0 on success, -1 on error. */ HWLOC_DECLSPEC int hwloc_bitmap_sscanf(hwloc_bitmap_t bitmap, const char * __hwloc_restrict string); /** \brief Stringify a bitmap in the list format. + * + * Note that if the bitmap is a CPU or nodeset, it contains physical indexes. * * Lists are comma-separated indexes or ranges. * Ranges are dash separated indexes. - * The last range may not have an ending indexes if the bitmap is infinitely set. + * A bitmap containing bits 1, 33, 34, and all from 64 to 95 is printed as "1,33-34,64-95". + * The last range may not have an ending index if the bitmap is infinitely set. * * Up to \p buflen characters may be written in buffer \p buf. * * If \p buflen is 0, \p buf may safely be \c NULL. * * \return the number of characters that were actually written if not truncating, - * or that would have been written (not including the ending \\0). + * or that would have been written (not including the ending \c \0). + * \return -1 on error. */ HWLOC_DECLSPEC int hwloc_bitmap_list_snprintf(char * __hwloc_restrict buf, size_t buflen, hwloc_const_bitmap_t bitmap); /** \brief Stringify a bitmap into a newly allocated list string. * - * \return 0 on success, -1 on error. + * Note that if the bitmap is a CPU or nodeset, it contains physical indexes. + * + * Lists are comma-separated indexes or ranges. + * Ranges are dash separated indexes. + * A bitmap containing bits 1, 33, 34, and all from 64 to 95 is printed as "1,33-34,64-95". + * The last range may not have an ending index if the bitmap is infinitely set. + * + * \return the number of characters that were written (not including the ending \c \0). + * \return -1 on error, for instance with \p errno set to \c ENOMEM on failure to allocate the output string. */ HWLOC_DECLSPEC int hwloc_bitmap_list_asprintf(char ** strp, hwloc_const_bitmap_t bitmap); /** \brief Parse a list string and stores it in bitmap \p bitmap. + * + * Note that if the bitmap is a CPU or nodeset, the input string must contain physical indexes. + * + * Lists are comma-separated indexes or ranges. + * Ranges are dash separated indexes. + * String "1,33-34,64-95" is parsed as a bitmap containing bits 1, 33, 34, and all from 64 to 95. + * The last range may not have an ending index if the bitmap is infinitely set. * * \return 0 on success, -1 on error. */ @@ -165,25 +202,43 @@ HWLOC_DECLSPEC int hwloc_bitmap_list_sscanf(hwloc_bitmap_t bitmap, const char * /** \brief Stringify a bitmap in the taskset-specific format. * - * The taskset command manipulates bitmap strings that contain a single + * Note that if the bitmap is a CPU or nodeset, it contains physical indexes. + * + * The taskset program manipulates bitmap strings that contain a single * (possible very long) hexadecimal number starting with 0x. + * A bitmap containing bits 1, 33, 34, and all from 64 to 95 is printed as "0xffffffff0000000600000002". * * Up to \p buflen characters may be written in buffer \p buf. * * If \p buflen is 0, \p buf may safely be \c NULL. * * \return the number of characters that were actually written if not truncating, - * or that would have been written (not including the ending \\0). + * or that would have been written (not including the ending \c \0). + * \return -1 on error. */ HWLOC_DECLSPEC int hwloc_bitmap_taskset_snprintf(char * __hwloc_restrict buf, size_t buflen, hwloc_const_bitmap_t bitmap); /** \brief Stringify a bitmap into a newly allocated taskset-specific string. * - * \return 0 on success, -1 on error. + * Note that if the bitmap is a CPU or nodeset, it contains physical indexes. + * + * The taskset program manipulates bitmap strings that contain a single + * (possible very long) hexadecimal number starting with 0x. + * A bitmap containing bits 1, 33, 34, and all from 64 to 95 is printed as "0xffffffff0000000600000002". + * + * \return the number of characters that were written (not including the ending \c \0). + * \return -1 on error, for instance with \p errno set to \c ENOMEM on failure to allocate the output string. */ HWLOC_DECLSPEC int hwloc_bitmap_taskset_asprintf(char ** strp, hwloc_const_bitmap_t bitmap); /** \brief Parse a taskset-specific bitmap string and stores it in bitmap \p bitmap. + * + * Note that if the bitmap is a CPU or nodeset, the input string must contain physical indexes. + * + * The taskset program manipulates bitmap strings that contain a single + * (possible very long) hexadecimal number starting with 0x. + * String "0xffffffff0000000600000002" is parsed as a bitmap containing all bits between 64 and 95, + * and bits 33, 34 and 1. * * \return 0 on success, -1 on error. */ diff --git a/src/3rdparty/hwloc/include/hwloc/diff.h b/src/3rdparty/hwloc/include/hwloc/diff.h index 4d822434..a9afa8f7 100644 --- a/src/3rdparty/hwloc/include/hwloc/diff.h +++ b/src/3rdparty/hwloc/include/hwloc/diff.h @@ -1,5 +1,5 @@ /* - * Copyright © 2013-2023 Inria. All rights reserved. + * Copyright © 2013-2024 Inria. All rights reserved. * See COPYING in top-level directory. */ @@ -258,7 +258,7 @@ HWLOC_DECLSPEC int hwloc_topology_diff_export_xml(hwloc_topology_diff_t diff, co /** \brief Load a list of topology differences from a XML buffer. * * Build a list of differences from the XML memory buffer given - * at \p xmlbuffer and of length \p buflen (including an ending \0). + * at \p xmlbuffer and of length \p buflen (including an ending \c \0). * This buffer may have been filled earlier with * hwloc_topology_diff_export_xmlbuffer(). * @@ -284,7 +284,7 @@ HWLOC_DECLSPEC int hwloc_topology_diff_load_xmlbuffer(const char *xmlbuffer, int * that contains the reference topology. * This attribute is given back when reading the diff from XML. * - * The returned buffer ends with a \0 that is included in the returned + * The returned buffer ends with a \c \0 that is included in the returned * length. * * \return 0 on success, -1 on error. diff --git a/src/3rdparty/hwloc/include/hwloc/distances.h b/src/3rdparty/hwloc/include/hwloc/distances.h index 71cca4b5..15b2dc8a 100644 --- a/src/3rdparty/hwloc/include/hwloc/distances.h +++ b/src/3rdparty/hwloc/include/hwloc/distances.h @@ -1,5 +1,5 @@ /* - * Copyright © 2010-2023 Inria. All rights reserved. + * Copyright © 2010-2025 Inria. All rights reserved. * See COPYING in top-level directory. */ @@ -28,18 +28,18 @@ extern "C" { /** \brief Matrix of distances between a set of objects. * - * This matrix often contains latencies between NUMA nodes + * The most common matrix contains latencies between NUMA nodes * (as reported in the System Locality Distance Information Table (SLIT) * in the ACPI specification), which may or may not be physically accurate. * It corresponds to the latency for accessing the memory of one node * from a core in another node. - * The corresponding kind is ::HWLOC_DISTANCES_KIND_FROM_OS | ::HWLOC_DISTANCES_KIND_FROM_USER. + * The corresponding kind is ::HWLOC_DISTANCES_KIND_MEANS_LATENCY | ::HWLOC_DISTANCES_KIND_FROM_USER. * The name of this distances structure is "NUMALatency". - * Others distance structures include and "XGMIBandwidth", "XGMIHops", - * "XeLinkBandwidth" and "NVLinkBandwidth". * * The matrix may also contain bandwidths between random sets of objects, * possibly provided by the user, as specified in the \p kind attribute. + * Others common distance structures include and "XGMIBandwidth", "XGMIHops", + * "XeLinkBandwidth" and "NVLinkBandwidth". * * Pointers \p objs and \p values should not be replaced, reallocated, freed, etc. * However callers are allowed to modify \p kind as well as the contents @@ -70,11 +70,10 @@ struct hwloc_distances_s { * The \p kind attribute of struct hwloc_distances_s is a OR'ed set * of kinds. * - * A kind of format HWLOC_DISTANCES_KIND_FROM_* specifies where the - * distance information comes from, if known. - * - * A kind of format HWLOC_DISTANCES_KIND_MEANS_* specifies whether - * values are latencies or bandwidths, if applicable. + * Each distance matrix may have only one kind among HWLOC_DISTANCES_KIND_FROM_* + * specifying where distance information comes from, + * and one kind among HWLOC_DISTANCES_KIND_MEANS_* specifying + * whether values are latencies or bandwidths. */ enum hwloc_distances_kind_e { /** \brief These distances were obtained from the operating system or hardware. @@ -228,17 +227,24 @@ enum hwloc_distances_transform_e { HWLOC_DISTANCES_TRANSFORM_LINKS = 1, /** \brief Merge switches with multiple ports into a single object. - * This currently only applies to NVSwitches where GPUs seem connected to different - * separate switch ports in the NVLinkBandwidth matrix. This transformation will - * replace all of them with the same port connected to all GPUs. - * Other ports are removed by applying ::HWLOC_DISTANCES_TRANSFORM_REMOVE_NULL internally. + * + * This currently only applies to NVSwitches where GPUs seem connected + * to different switch ports. Switch ports must be objects with subtype + * "NVSwitch" as in the NVLinkBandwidth matrix. + * + * This transformation will replace all ports with only the first one, + * now connected to all GPUs. Other ports are removed by applying + * ::HWLOC_DISTANCES_TRANSFORM_REMOVE_NULL internally. * \hideinitializer */ HWLOC_DISTANCES_TRANSFORM_MERGE_SWITCH_PORTS = 2, /** \brief Apply a transitive closure to the matrix to connect objects across switches. - * This currently only applies to GPUs and NVSwitches in the NVLinkBandwidth matrix. - * All pairs of GPUs will be reported as directly connected. + * + * All pairs of GPUs will be reported as directly connected instead GPUs being + * only connected to switches. + * + * Switch ports must be objects with subtype "NVSwitch" as in the NVLinkBandwidth matrix. * \hideinitializer */ HWLOC_DISTANCES_TRANSFORM_TRANSITIVE_CLOSURE = 3 @@ -357,6 +363,8 @@ typedef void * hwloc_distances_add_handle_t; * Otherwise, it will be copied internally and may later be freed by the caller. * * \p kind specifies the kind of distance as a OR'ed set of ::hwloc_distances_kind_e. + * Only one kind of meaning and one kind of provenance may be given if appropriate + * (e.g. ::HWLOC_DISTANCES_KIND_MEANS_BANDWIDTH and ::HWLOC_DISTANCES_KIND_FROM_USER). * Kind ::HWLOC_DISTANCES_KIND_HETEROGENEOUS_TYPES will be automatically set * according to objects having different types in hwloc_distances_add_values(). * @@ -403,7 +411,8 @@ HWLOC_DECLSPEC int hwloc_distances_add_values(hwloc_topology_t topology, /** \brief Flags for adding a new distances to a topology. */ enum hwloc_distances_add_flag_e { /** \brief Try to group objects based on the newly provided distance information. - * This is ignored for distances between objects of different types. + * Grouping is only performed when the distances structure contains latencies, + * and when all objects are of the same type. * \hideinitializer */ HWLOC_DISTANCES_ADD_FLAG_GROUP = (1UL<<0), diff --git a/src/3rdparty/hwloc/include/hwloc/helper.h b/src/3rdparty/hwloc/include/hwloc/helper.h index 01619c5f..01640fb7 100644 --- a/src/3rdparty/hwloc/include/hwloc/helper.h +++ b/src/3rdparty/hwloc/include/hwloc/helper.h @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2023 Inria. All rights reserved. + * Copyright © 2009-2024 Inria. All rights reserved. * Copyright © 2009-2012 Université Bordeaux * Copyright © 2009-2010 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -946,6 +946,14 @@ enum hwloc_distrib_flags_e { * * \return 0 on success, -1 on error. * + * \note On hybrid CPUs (or asymmetric platforms), distribution may be suboptimal + * since the number of cores or PUs inside packages or below caches may vary + * (the top-down recursive partitioning ignores these numbers until reaching their levels). + * Hence it is recommended to distribute only inside a single homogeneous domain. + * For instance on a CPU with energy-efficient E-cores and high-performance P-cores, + * one should distribute separately N tasks on E-cores and M tasks on P-cores + * instead of trying to distribute directly M+N tasks on the entire CPUs. + * * \note This function requires the \p roots objects to have a CPU set. */ static __hwloc_inline int @@ -960,7 +968,7 @@ hwloc_distrib(hwloc_topology_t topology, unsigned given, givenweight; hwloc_cpuset_t *cpusetp = set; - if (flags & ~HWLOC_DISTRIB_FLAG_REVERSE) { + if (!n || (flags & ~HWLOC_DISTRIB_FLAG_REVERSE)) { errno = EINVAL; return -1; } diff --git a/src/3rdparty/hwloc/include/hwloc/levelzero.h b/src/3rdparty/hwloc/include/hwloc/levelzero.h index dcdcf1fb..c1b43845 100644 --- a/src/3rdparty/hwloc/include/hwloc/levelzero.h +++ b/src/3rdparty/hwloc/include/hwloc/levelzero.h @@ -1,5 +1,5 @@ /* - * Copyright © 2021-2023 Inria. All rights reserved. + * Copyright © 2021-2024 Inria. All rights reserved. * See COPYING in top-level directory. */ @@ -32,7 +32,8 @@ extern "C" { /** \defgroup hwlocality_levelzero Interoperability with the oneAPI Level Zero interface. * * This interface offers ways to retrieve topology information about - * devices managed by the Level Zero API. + * devices managed by the Level Zero API, both for main Core devices (ZE API) + * and the Sysman devices (ZES API). * * @{ */ @@ -44,9 +45,68 @@ extern "C" { * the Level Zero device \p device. * * Topology \p topology and device \p device must match the local machine. + * The Level Zero library must have been initialized with zeInit(). + * I/O devices detection and the Level Zero component are not needed in the + * topology. + * + * The function only returns the locality of the device. + * If more information about the device is needed, OS objects should + * be used instead, see hwloc_levelzero_get_device_osdev(). + * + * This function is currently only implemented in a meaningful way for + * Linux; other systems will simply get a full cpuset. + * + * \return 0 on success. + * \return -1 on error, for instance if device information could not be found. + * + * \note zeDevicePciGetPropertiesExt() must be supported, or the entire machine + * locality will be returned. + */ +static __hwloc_inline int +hwloc_levelzero_get_device_cpuset(hwloc_topology_t topology __hwloc_attribute_unused, + ze_device_handle_t device, hwloc_cpuset_t set) +{ +#ifdef HWLOC_LINUX_SYS + /* If we're on Linux, use the sysfs mechanism to get the local cpus */ +#define HWLOC_LEVELZERO_DEVICE_SYSFS_PATH_MAX 128 + char path[HWLOC_LEVELZERO_DEVICE_SYSFS_PATH_MAX]; + ze_pci_ext_properties_t pci; + ze_result_t res; + + if (!hwloc_topology_is_thissystem(topology)) { + errno = EINVAL; + return -1; + } + + pci.stype = ZE_STRUCTURE_TYPE_PCI_EXT_PROPERTIES; + pci.pNext = NULL; + res = zeDevicePciGetPropertiesExt(device, &pci); + if (res != ZE_RESULT_SUCCESS) { + errno = EINVAL; + return -1; + } + + sprintf(path, "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/local_cpus", + pci.address.domain, pci.address.bus, pci.address.device, pci.address.function); + if (hwloc_linux_read_path_as_cpumask(path, set) < 0 + || hwloc_bitmap_iszero(set)) + hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology)); +#else + /* Non-Linux systems simply get a full cpuset */ + hwloc_bitmap_copy(set, hwloc_topology_get_complete_cpuset(topology)); +#endif + return 0; +} + +/** \brief Get the CPU set of logical processors that are physically + * close to the Level Zero Sysman device \p device + * + * Store in \p set the CPU-set describing the locality of + * the Level Zero device \p device. + * + * Topology \p topology and device \p device must match the local machine. * The Level Zero library must have been initialized with Sysman enabled - * (by calling zesInit(0) if supported, - * or by setting ZES_ENABLE_SYSMAN=1 in the environment). + * with zesInit(). * I/O devices detection and the Level Zero component are not needed in the * topology. * @@ -61,15 +121,14 @@ extern "C" { * \return -1 on error, for instance if device information could not be found. */ static __hwloc_inline int -hwloc_levelzero_get_device_cpuset(hwloc_topology_t topology __hwloc_attribute_unused, - ze_device_handle_t device, hwloc_cpuset_t set) +hwloc_levelzero_get_sysman_device_cpuset(hwloc_topology_t topology __hwloc_attribute_unused, + zes_device_handle_t device, hwloc_cpuset_t set) { #ifdef HWLOC_LINUX_SYS /* If we're on Linux, use the sysfs mechanism to get the local cpus */ #define HWLOC_LEVELZERO_DEVICE_SYSFS_PATH_MAX 128 char path[HWLOC_LEVELZERO_DEVICE_SYSFS_PATH_MAX]; zes_pci_properties_t pci; - zes_device_handle_t sdevice = device; ze_result_t res; if (!hwloc_topology_is_thissystem(topology)) { @@ -77,7 +136,7 @@ hwloc_levelzero_get_device_cpuset(hwloc_topology_t topology __hwloc_attribute_un return -1; } - res = zesDevicePciGetProperties(sdevice, &pci); + res = zesDevicePciGetProperties(device, &pci); if (res != ZE_RESULT_SUCCESS) { errno = EINVAL; return -1; @@ -102,17 +161,90 @@ hwloc_levelzero_get_device_cpuset(hwloc_topology_t topology __hwloc_attribute_un * \return \c NULL if none could be found. * * Topology \p topology and device \p dv_ind must match the local machine. + * The Level Zero library must have been initialized with zeInit(). * I/O devices detection and the Level Zero component must be enabled in the * topology. If not, the locality of the object may still be found using * hwloc_levelzero_get_device_cpuset(). * + * \note If the input ZE device is actually a subdevice, then its parent + * (root device) is actually translated, i.e. the main hwloc OS device + * is returned instead of one of its children. + * + * \note The corresponding hwloc PCI device may be found by looking + * at the result parent pointer (unless PCI devices are filtered out). + * + * \note zeDevicePciGetPropertiesExt() must be supported. + */ +static __hwloc_inline hwloc_obj_t +hwloc_levelzero_get_device_osdev(hwloc_topology_t topology, ze_device_handle_t device) +{ + ze_pci_ext_properties_t pci; + ze_result_t res; + hwloc_obj_t osdev; + + if (!hwloc_topology_is_thissystem(topology)) { + errno = EINVAL; + return NULL; + } + + pci.stype = ZE_STRUCTURE_TYPE_PCI_EXT_PROPERTIES; + pci.pNext = NULL; + res = zeDevicePciGetPropertiesExt(device, &pci); + if (res != ZE_RESULT_SUCCESS) { + errno = EINVAL; + return NULL; + } + + osdev = NULL; + while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) { + hwloc_obj_t pcidev; + + if (strncmp(osdev->name, "ze", 2)) + continue; + + pcidev = osdev; + while (pcidev && pcidev->type != HWLOC_OBJ_PCI_DEVICE) + pcidev = pcidev->parent; + if (!pcidev) + continue; + + if (pcidev + && pcidev->type == HWLOC_OBJ_PCI_DEVICE + && pcidev->attr->pcidev.domain == pci.address.domain + && pcidev->attr->pcidev.bus == pci.address.bus + && pcidev->attr->pcidev.dev == pci.address.device + && pcidev->attr->pcidev.func == pci.address.function) + return osdev; + + /* FIXME: when we'll have serialnumber, try it in case PCI is filtered-out */ + } + + return NULL; +} + +/** \brief Get the hwloc OS device object corresponding to Level Zero Sysman device + * \p device. + * + * \return The hwloc OS device object that describes the given Level Zero device \p device. + * \return \c NULL if none could be found. + * + * Topology \p topology and device \p dv_ind must match the local machine. + * The Level Zero library must have been initialized with Sysman enabled + * with zesInit(). + * I/O devices detection and the Level Zero component must be enabled in the + * topology. If not, the locality of the object may still be found using + * hwloc_levelzero_get_device_cpuset(). + * + * \note If the input ZES device is actually a subdevice, then its parent + * (root device) is actually translated, i.e. the main hwloc OS device + * is returned instead of one of its children. + * * \note The corresponding hwloc PCI device may be found by looking * at the result parent pointer (unless PCI devices are filtered out). */ static __hwloc_inline hwloc_obj_t -hwloc_levelzero_get_device_osdev(hwloc_topology_t topology, ze_device_handle_t device) +hwloc_levelzero_get_sysman_device_osdev(hwloc_topology_t topology, zes_device_handle_t device) { - zes_device_handle_t sdevice = device; zes_pci_properties_t pci; ze_result_t res; hwloc_obj_t osdev; @@ -122,20 +254,25 @@ hwloc_levelzero_get_device_osdev(hwloc_topology_t topology, ze_device_handle_t d return NULL; } - res = zesDevicePciGetProperties(sdevice, &pci); + res = zesDevicePciGetProperties(device, &pci); if (res != ZE_RESULT_SUCCESS) { - /* L0 was likely initialized without sysman, don't bother */ errno = EINVAL; return NULL; } osdev = NULL; while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) { - hwloc_obj_t pcidev = osdev->parent; + hwloc_obj_t pcidev; if (strncmp(osdev->name, "ze", 2)) continue; + pcidev = osdev; + while (pcidev && pcidev->type != HWLOC_OBJ_PCI_DEVICE) + pcidev = pcidev->parent; + if (!pcidev) + continue; + if (pcidev && pcidev->type == HWLOC_OBJ_PCI_DEVICE && pcidev->attr->pcidev.domain == pci.address.domain diff --git a/src/3rdparty/hwloc/include/hwloc/memattrs.h b/src/3rdparty/hwloc/include/hwloc/memattrs.h index 10332b8e..ad9278cc 100644 --- a/src/3rdparty/hwloc/include/hwloc/memattrs.h +++ b/src/3rdparty/hwloc/include/hwloc/memattrs.h @@ -1,5 +1,5 @@ /* - * Copyright © 2019-2023 Inria. All rights reserved. + * Copyright © 2019-2025 Inria. All rights reserved. * See COPYING in top-level directory. */ @@ -58,6 +58,11 @@ extern "C" { * an easy way to distinguish NUMA nodes of different kinds, as explained * in \ref heteromem. * + * Beside tiers, hwloc defines a set of "default" nodes where normal memory + * allocations should be made from (see hwloc_topology_get_default_nodeset()). + * This is also useful for dividing the machine into a set of non-overlapping + * NUMA domains, for instance for binding tasks per domain. + * * \sa An example is available in doc/examples/memory-attributes.c in the source tree. * * \note The API also supports specific objects as initiator, @@ -69,7 +74,10 @@ extern "C" { * @{ */ -/** \brief Memory node attributes. */ +/** \brief Predefined memory attribute IDs. + * See ::hwloc_memattr_id_t for the generic definition of IDs + * for predefined or custom attributes. + */ enum hwloc_memattr_id_e { /** \brief * The \"Capacity\" is returned in bytes (local_memory attribute in objects). @@ -78,6 +86,8 @@ enum hwloc_memattr_id_e { * * No initiator is involved when looking at this attribute. * The corresponding attribute flags are ::HWLOC_MEMATTR_FLAG_HIGHER_FIRST. + * + * Capacity values may not be modified using hwloc_memattr_set_value(). * \hideinitializer */ HWLOC_MEMATTR_ID_CAPACITY = 0, @@ -93,6 +103,8 @@ enum hwloc_memattr_id_e { * * No initiator is involved when looking at this attribute. * The corresponding attribute flags are ::HWLOC_MEMATTR_FLAG_HIGHER_FIRST. + + * Locality values may not be modified using hwloc_memattr_set_value(). * \hideinitializer */ HWLOC_MEMATTR_ID_LOCALITY = 1, @@ -173,11 +185,19 @@ enum hwloc_memattr_id_e { /* TODO persistence? */ - HWLOC_MEMATTR_ID_MAX /**< \private Sentinel value */ + HWLOC_MEMATTR_ID_MAX /**< \private + * Sentinel value for predefined attributes. + * Dynamically registered custom attributes start here. + */ }; /** \brief A memory attribute identifier. - * May be either one of ::hwloc_memattr_id_e or a new id returned by hwloc_memattr_register(). + * + * hwloc predefines some commonly-used attributes in ::hwloc_memattr_id_e. + * One may then dynamically register custom ones with hwloc_memattr_register(), + * they will be assigned IDs immediately after the predefined ones. + * See \ref hwlocality_memattrs_manage for more information about + * existing attribute IDs. */ typedef unsigned hwloc_memattr_id_t; @@ -230,6 +250,16 @@ enum hwloc_local_numanode_flag_e { */ HWLOC_LOCAL_NUMANODE_FLAG_SMALLER_LOCALITY = (1UL<<1), + /** \breif Select NUMA nodes whose locality intersects the given cpuset. + * This includes larger and smaller localities as well as localities + * that are partially included. + * For instance, if the locality is one core of both packages, a NUMA node + * local to one package is neither larger nor smaller than this locality, + * but it intersects it. + * \hideinitializer + */ + HWLOC_LOCAL_NUMANODE_FLAG_INTERSECT_LOCALITY = (1UL<<3), + /** \brief Select all NUMA nodes in the topology. * The initiator \p initiator is ignored. * \hideinitializer @@ -275,7 +305,57 @@ hwloc_get_local_numanode_objs(hwloc_topology_t topology, hwloc_obj_t *nodes, unsigned long flags); - +/** \brief Return the set of default NUMA nodes + * + * In machines with heterogeneous memory, some NUMA nodes are considered + * the default ones, i.e. where basic allocations should be made from. + * These are usually DRAM nodes. + * + * Other nodes may be reserved for specific use (I/O device memory, e.g. GPU memory), + * small but high performance (HBM), large but slow memory (NVM), etc. + * Buffers should usually not be allocated from there unless explicitly required. + * + * This function fills \p nodeset with the bits of NUMA nodes considered default. + * + * It is guaranteed that these nodes have non-intersecting CPU sets, + * i.e. cores may not have multiple local NUMA nodes anymore. + * Hence this may be used to iterate over the platform divided into separate + * NUMA localities, for instance for binding one task per NUMA domain. + * + * Any core that had some local NUMA node(s) in the initial topology should + * still have one in the default nodeset. Corner cases where this would be + * wrong consist in asymmetric platforms with missing DRAM nodes, or topologies + * that were already restricted to less NUMA nodes. + * + * The returned nodeset may be passed to hwloc_topology_restrict() with + * ::HWLOC_RESTRICT_FLAG_BYNODESET to remove all non-default nodes from + * the topology. The resulting topology will be easier to use when iterating + * over (now homogeneous) NUMA nodes. + * + * The heuristics for finding default nodes relies on memory tiers and subtypes + * (see \ref heteromem) as well as the assumption that hardware vendors list + * default nodes first in hardware tables. + * + * \p flags must be \c 0 for now. + * + * \return 0 on success. + * \return -1 on error. + * + * \note The returned nodeset usually contains all nodes from a single memory + * tier, likely the DRAM one. + * + * \note The returned nodeset is included in the list of available nodes + * returned by hwloc_topology_get_topology_nodeset(). It is strictly smaller + * if the machine has heterogeneous memory. + * + * \note The heuristics may return a suboptimal set of nodes if hwloc could + * not guess memory types and/or if some default nodes were removed earlier + * from the topology (e.g. with hwloc_topology_restrict()). + */ +HWLOC_DECLSPEC int +hwloc_topology_get_default_nodeset(hwloc_topology_t topology, + hwloc_nodeset_t nodeset, + unsigned long flags); /** \brief Return an attribute value for a specific target NUMA node. * @@ -283,6 +363,10 @@ hwloc_get_local_numanode_objs(hwloc_topology_t topology, * (it does not have the flag ::HWLOC_MEMATTR_FLAG_NEED_INITIATOR), * location \p initiator is ignored and may be \c NULL. * + * \p target_node cannot be \c NULL. If \p attribute is ::HWLOC_MEMATTR_ID_CAPACITY, + * \p target_node must be a NUMA node. If it is ::HWLOC_MEMATTR_ID_LOCALITY, + * \p target_node must have a CPU set. + * * \p flags must be \c 0 for now. * * \return 0 on success. @@ -352,6 +436,8 @@ hwloc_memattr_get_best_target(hwloc_topology_t topology, * The returned initiator should not be modified or freed, * it belongs to the topology. * + * \p target_node cannot be \c NULL. + * * \p flags must be \c 0 for now. * * \return 0 on success. @@ -362,100 +448,10 @@ hwloc_memattr_get_best_target(hwloc_topology_t topology, HWLOC_DECLSPEC int hwloc_memattr_get_best_initiator(hwloc_topology_t topology, hwloc_memattr_id_t attribute, - hwloc_obj_t target, + hwloc_obj_t target_node, unsigned long flags, struct hwloc_location *best_initiator, hwloc_uint64_t *value); -/** @} */ - - -/** \defgroup hwlocality_memattrs_manage Managing memory attributes - * @{ - */ - -/** \brief Return the name of a memory attribute. - * - * \return 0 on success. - * \return -1 with errno set to \c EINVAL if the attribute does not exist. - */ -HWLOC_DECLSPEC int -hwloc_memattr_get_name(hwloc_topology_t topology, - hwloc_memattr_id_t attribute, - const char **name); - -/** \brief Return the flags of the given attribute. - * - * Flags are a OR'ed set of ::hwloc_memattr_flag_e. - * - * \return 0 on success. - * \return -1 with errno set to \c EINVAL if the attribute does not exist. - */ -HWLOC_DECLSPEC int -hwloc_memattr_get_flags(hwloc_topology_t topology, - hwloc_memattr_id_t attribute, - unsigned long *flags); - -/** \brief Memory attribute flags. - * Given to hwloc_memattr_register() and returned by hwloc_memattr_get_flags(). - */ -enum hwloc_memattr_flag_e { - /** \brief The best nodes for this memory attribute are those with the higher values. - * For instance Bandwidth. - */ - HWLOC_MEMATTR_FLAG_HIGHER_FIRST = (1UL<<0), - /** \brief The best nodes for this memory attribute are those with the lower values. - * For instance Latency. - */ - HWLOC_MEMATTR_FLAG_LOWER_FIRST = (1UL<<1), - /** \brief The value returned for this memory attribute depends on the given initiator. - * For instance Bandwidth and Latency, but not Capacity. - */ - HWLOC_MEMATTR_FLAG_NEED_INITIATOR = (1UL<<2) -}; - -/** \brief Register a new memory attribute. - * - * Add a specific memory attribute that is not defined in ::hwloc_memattr_id_e. - * Flags are a OR'ed set of ::hwloc_memattr_flag_e. It must contain at least - * one of ::HWLOC_MEMATTR_FLAG_HIGHER_FIRST or ::HWLOC_MEMATTR_FLAG_LOWER_FIRST. - * - * \return 0 on success. - * \return -1 with errno set to \c EBUSY if another attribute already uses this name. - */ -HWLOC_DECLSPEC int -hwloc_memattr_register(hwloc_topology_t topology, - const char *name, - unsigned long flags, - hwloc_memattr_id_t *id); - -/** \brief Set an attribute value for a specific target NUMA node. - * - * If the attribute does not relate to a specific initiator - * (it does not have the flag ::HWLOC_MEMATTR_FLAG_NEED_INITIATOR), - * location \p initiator is ignored and may be \c NULL. - * - * The initiator will be copied into the topology, - * the caller should free anything allocated to store the initiator, - * for instance the cpuset. - * - * \p flags must be \c 0 for now. - * - * \note The initiator \p initiator should be of type ::HWLOC_LOCATION_TYPE_CPUSET - * when referring to accesses performed by CPU cores. - * ::HWLOC_LOCATION_TYPE_OBJECT is currently unused internally by hwloc, - * but users may for instance use it to provide custom information about - * host memory accesses performed by GPUs. - * - * \return 0 on success or -1 on error. - */ -HWLOC_DECLSPEC int -hwloc_memattr_set_value(hwloc_topology_t topology, - hwloc_memattr_id_t attribute, - hwloc_obj_t target_node, - struct hwloc_location *initiator, - unsigned long flags, - hwloc_uint64_t value); - /** \brief Return the target NUMA nodes that have some values for a given attribute. * * Return targets for the given attribute in the \p targets array @@ -519,6 +515,8 @@ hwloc_memattr_get_targets(hwloc_topology_t topology, * The returned initiators should not be modified or freed, * they belong to the topology. * + * \p target_node cannot be \c NULL. + * * \p flags must be \c 0 for now. * * If the attribute does not relate to a specific initiator @@ -538,6 +536,131 @@ hwloc_memattr_get_initiators(hwloc_topology_t topology, hwloc_obj_t target_node, unsigned long flags, unsigned *nr, struct hwloc_location *initiators, hwloc_uint64_t *values); + +/** @} */ + + +/** \defgroup hwlocality_memattrs_manage Managing memory attributes + * + * Memory attribues are identified by an ID (::hwloc_memattr_id_t) + * and a name. hwloc_memattr_get_name() and hwloc_memattr_get_by_name() + * convert between them (or return error if the attribute does not exist). + * + * The set of valid ::hwloc_memattr_id_t is a contigous set starting at \c 0. + * It first contains predefined attributes, as listed + * in ::hwloc_memattr_id_e (from \c 0 to \c HWLOC_MEMATTR_ID_MAX-1). + * Then custom attributes may be dynamically registered with + * hwloc_memattr_register(). They will get the following IDs + * (\c HWLOC_MEMATTR_ID_MAX for the first one, etc.). + * + * To iterate over all valid attributes + * (either predefined or dynamically registered custom ones), + * one may iterate over IDs starting from \c 0 until hwloc_memattr_get_name() + * or hwloc_memattr_get_flags() returns an error. + * + * The values for an existing attribute or for custom dynamically registered ones + * may be set or modified with hwloc_memattr_set_value(). + * + * @{ + */ + +/** \brief Return the name of a memory attribute. + * + * The output pointer \p name cannot be \c NULL. + * + * \return 0 on success. + * \return -1 with errno set to \c EINVAL if the attribute does not exist. + */ +HWLOC_DECLSPEC int +hwloc_memattr_get_name(hwloc_topology_t topology, + hwloc_memattr_id_t attribute, + const char **name); + +/** \brief Return the flags of the given attribute. + * + * Flags are a OR'ed set of ::hwloc_memattr_flag_e. + * + * The output pointer \p flags cannot be \c NULL. + * + * \return 0 on success. + * \return -1 with errno set to \c EINVAL if the attribute does not exist. + */ +HWLOC_DECLSPEC int +hwloc_memattr_get_flags(hwloc_topology_t topology, + hwloc_memattr_id_t attribute, + unsigned long *flags); + +/** \brief Memory attribute flags. + * Given to hwloc_memattr_register() and returned by hwloc_memattr_get_flags(). + */ +enum hwloc_memattr_flag_e { + /** \brief The best nodes for this memory attribute are those with the higher values. + * For instance Bandwidth. + */ + HWLOC_MEMATTR_FLAG_HIGHER_FIRST = (1UL<<0), + /** \brief The best nodes for this memory attribute are those with the lower values. + * For instance Latency. + */ + HWLOC_MEMATTR_FLAG_LOWER_FIRST = (1UL<<1), + /** \brief The value returned for this memory attribute depends on the given initiator. + * For instance Bandwidth and Latency, but not Capacity. + */ + HWLOC_MEMATTR_FLAG_NEED_INITIATOR = (1UL<<2) +}; + +/** \brief Register a new memory attribute. + * + * Add a new custom memory attribute. + * Flags are a OR'ed set of ::hwloc_memattr_flag_e. It must contain one of + * ::HWLOC_MEMATTR_FLAG_HIGHER_FIRST or ::HWLOC_MEMATTR_FLAG_LOWER_FIRST but not both. + * + * The new attribute \p id is immediately after the last existing attribute ID + * (which is either the ID of the last registered attribute if any, + * or the ID of the last predefined attribute in ::hwloc_memattr_id_e). + * + * \return 0 on success. + * \return -1 with errno set to \c EINVAL if an invalid set of flags is given. + * \return -1 with errno set to \c EBUSY if another attribute already uses this name. + */ +HWLOC_DECLSPEC int +hwloc_memattr_register(hwloc_topology_t topology, + const char *name, + unsigned long flags, + hwloc_memattr_id_t *id); + +/** \brief Set an attribute value for a specific target NUMA node. + * + * If the attribute does not relate to a specific initiator + * (it does not have the flag ::HWLOC_MEMATTR_FLAG_NEED_INITIATOR), + * location \p initiator is ignored and may be \c NULL. + * + * The initiator will be copied into the topology, + * the caller should free anything allocated to store the initiator, + * for instance the cpuset. + * + * \p target_node cannot be \c NULL. + * + * \p attribute cannot be ::HWLOC_MEMATTR_FLAG_ID_CAPACITY or + * ::HWLOC_MEMATTR_FLAG_ID_LOCALITY. + * + * \p flags must be \c 0 for now. + * + * \note The initiator \p initiator should be of type ::HWLOC_LOCATION_TYPE_CPUSET + * when referring to accesses performed by CPU cores. + * ::HWLOC_LOCATION_TYPE_OBJECT is currently unused internally by hwloc, + * but users may for instance use it to provide custom information about + * host memory accesses performed by GPUs. + * + * \return 0 on success or -1 on error. + */ +HWLOC_DECLSPEC int +hwloc_memattr_set_value(hwloc_topology_t topology, + hwloc_memattr_id_t attribute, + hwloc_obj_t target_node, + struct hwloc_location *initiator, + unsigned long flags, + hwloc_uint64_t value); + /** @} */ #ifdef __cplusplus diff --git a/src/3rdparty/hwloc/include/hwloc/opencl.h b/src/3rdparty/hwloc/include/hwloc/opencl.h index 9810504e..5e53b2aa 100644 --- a/src/3rdparty/hwloc/include/hwloc/opencl.h +++ b/src/3rdparty/hwloc/include/hwloc/opencl.h @@ -41,6 +41,15 @@ extern "C" { */ /* Copyright (c) 2008-2018 The Khronos Group Inc. */ +/* needs "cl_khr_pci_bus_info" device extension, but not strictly required for clGetDeviceInfo() */ +typedef struct { + cl_uint pci_domain; + cl_uint pci_bus; + cl_uint pci_device; + cl_uint pci_function; +} hwloc_cl_device_pci_bus_info_khr; +#define HWLOC_CL_DEVICE_PCI_BUS_INFO_KHR 0x410F + /* needs "cl_amd_device_attribute_query" device extension, but not strictly required for clGetDeviceInfo() */ #define HWLOC_CL_DEVICE_TOPOLOGY_AMD 0x4037 typedef union { @@ -78,9 +87,19 @@ hwloc_opencl_get_device_pci_busid(cl_device_id device, unsigned *domain, unsigned *bus, unsigned *dev, unsigned *func) { hwloc_cl_device_topology_amd amdtopo; + hwloc_cl_device_pci_bus_info_khr khrbusinfo; cl_uint nvbus, nvslot, nvdomain; cl_int clret; + clret = clGetDeviceInfo(device, HWLOC_CL_DEVICE_PCI_BUS_INFO_KHR, sizeof(khrbusinfo), &khrbusinfo, NULL); + if (CL_SUCCESS == clret) { + *domain = (unsigned) khrbusinfo.pci_domain; + *bus = (unsigned) khrbusinfo.pci_bus; + *dev = (unsigned) khrbusinfo.pci_device; + *func = (unsigned) khrbusinfo.pci_function; + return 0; + } + clret = clGetDeviceInfo(device, HWLOC_CL_DEVICE_TOPOLOGY_AMD, sizeof(amdtopo), &amdtopo, NULL); if (CL_SUCCESS == clret && HWLOC_CL_DEVICE_TOPOLOGY_TYPE_PCIE_AMD == amdtopo.raw.type) { diff --git a/src/3rdparty/hwloc/include/hwloc/plugins.h b/src/3rdparty/hwloc/include/hwloc/plugins.h index f3db648c..96f95f1f 100644 --- a/src/3rdparty/hwloc/include/hwloc/plugins.h +++ b/src/3rdparty/hwloc/include/hwloc/plugins.h @@ -1,5 +1,5 @@ /* - * Copyright © 2013-2022 Inria. All rights reserved. + * Copyright © 2013-2024 Inria. All rights reserved. * Copyright © 2016 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. */ @@ -26,7 +26,7 @@ struct hwloc_backend; -/** \defgroup hwlocality_disc_components Components and Plugins: Discovery components +/** \defgroup hwlocality_disc_components Components and Plugins: Discovery components and backends * * \note These structures and functions may change when ::HWLOC_COMPONENT_ABI is modified. * @@ -90,18 +90,6 @@ struct hwloc_disc_component { struct hwloc_disc_component * next; }; -/** @} */ - - - - -/** \defgroup hwlocality_disc_backends Components and Plugins: Discovery backends - * - * \note These structures and functions may change when ::HWLOC_COMPONENT_ABI is modified. - * - * @{ - */ - /** \brief Discovery phase */ typedef enum hwloc_disc_phase_e { /** \brief xml or synthetic, platform-specific components such as bgq. @@ -313,6 +301,64 @@ struct hwloc_component { void * data; }; +/** \brief Make sure that plugins can lookup core symbols. + * + * This is a sanity check to avoid lazy-lookup failures when libhwloc + * is loaded within a plugin, and later tries to load its own plugins. + * This may fail (and abort the program) if libhwloc symbols are in a + * private namespace. + * + * \return 0 on success. + * \return -1 if the plugin cannot be successfully loaded. The caller + * plugin init() callback should return a negative error code as well. + * + * Plugins should call this function in their init() callback to avoid + * later crashes if lazy symbol resolution is used by the upper layer that + * loaded hwloc (e.g. OpenCL implementations using dlopen with RTLD_LAZY). + * + * \note The build system must define HWLOC_INSIDE_PLUGIN if and only if + * building the caller as a plugin. + * + * \note This function should remain inline so plugins can call it even + * when they cannot find libhwloc symbols. + */ +static __hwloc_inline int +hwloc_plugin_check_namespace(const char *pluginname __hwloc_attribute_unused, const char *symbol __hwloc_attribute_unused) +{ +#ifdef HWLOC_INSIDE_PLUGIN + void *sym; +#ifdef HWLOC_HAVE_LTDL + lt_dlhandle handle = lt_dlopen(NULL); +#else + void *handle = dlopen(NULL, RTLD_NOW|RTLD_LOCAL); +#endif + if (!handle) + /* cannot check, assume things will work */ + return 0; +#ifdef HWLOC_HAVE_LTDL + sym = lt_dlsym(handle, symbol); + lt_dlclose(handle); +#else + sym = dlsym(handle, symbol); + dlclose(handle); +#endif + if (!sym) { + static int verboseenv_checked = 0; + static int verboseenv_value = 0; + if (!verboseenv_checked) { + const char *verboseenv = getenv("HWLOC_PLUGINS_VERBOSE"); + verboseenv_value = verboseenv ? atoi(verboseenv) : 0; + verboseenv_checked = 1; + } + if (verboseenv_value) + fprintf(stderr, "Plugin `%s' disabling itself because it cannot find the `%s' core symbol.\n", + pluginname, symbol); + return -1; + } +#endif /* HWLOC_INSIDE_PLUGIN */ + return 0; +} + /** @} */ @@ -422,64 +468,6 @@ HWLOC_DECLSPEC int hwloc_obj_add_children_sets(hwloc_obj_t obj); */ HWLOC_DECLSPEC int hwloc_topology_reconnect(hwloc_topology_t topology, unsigned long flags __hwloc_attribute_unused); -/** \brief Make sure that plugins can lookup core symbols. - * - * This is a sanity check to avoid lazy-lookup failures when libhwloc - * is loaded within a plugin, and later tries to load its own plugins. - * This may fail (and abort the program) if libhwloc symbols are in a - * private namespace. - * - * \return 0 on success. - * \return -1 if the plugin cannot be successfully loaded. The caller - * plugin init() callback should return a negative error code as well. - * - * Plugins should call this function in their init() callback to avoid - * later crashes if lazy symbol resolution is used by the upper layer that - * loaded hwloc (e.g. OpenCL implementations using dlopen with RTLD_LAZY). - * - * \note The build system must define HWLOC_INSIDE_PLUGIN if and only if - * building the caller as a plugin. - * - * \note This function should remain inline so plugins can call it even - * when they cannot find libhwloc symbols. - */ -static __hwloc_inline int -hwloc_plugin_check_namespace(const char *pluginname __hwloc_attribute_unused, const char *symbol __hwloc_attribute_unused) -{ -#ifdef HWLOC_INSIDE_PLUGIN - void *sym; -#ifdef HWLOC_HAVE_LTDL - lt_dlhandle handle = lt_dlopen(NULL); -#else - void *handle = dlopen(NULL, RTLD_NOW|RTLD_LOCAL); -#endif - if (!handle) - /* cannot check, assume things will work */ - return 0; -#ifdef HWLOC_HAVE_LTDL - sym = lt_dlsym(handle, symbol); - lt_dlclose(handle); -#else - sym = dlsym(handle, symbol); - dlclose(handle); -#endif - if (!sym) { - static int verboseenv_checked = 0; - static int verboseenv_value = 0; - if (!verboseenv_checked) { - const char *verboseenv = getenv("HWLOC_PLUGINS_VERBOSE"); - verboseenv_value = verboseenv ? atoi(verboseenv) : 0; - verboseenv_checked = 1; - } - if (verboseenv_value) - fprintf(stderr, "Plugin `%s' disabling itself because it cannot find the `%s' core symbol.\n", - pluginname, symbol); - return -1; - } -#endif /* HWLOC_INSIDE_PLUGIN */ - return 0; -} - /** @} */ @@ -645,6 +633,19 @@ HWLOC_DECLSPEC struct hwloc_obj * hwloc_pci_find_parent_by_busid(struct hwloc_to */ HWLOC_DECLSPEC struct hwloc_obj * hwloc_pci_find_by_busid(struct hwloc_topology *topology, unsigned domain, unsigned bus, unsigned dev, unsigned func); + +/** @} */ + + + + +/** \defgroup hwlocality_components_distances Components and Plugins: distances + * + * \note These structures and functions may change when ::HWLOC_COMPONENT_ABI is modified. + * + * @{ + */ + /** \brief Handle to a new distances structure during its addition to the topology. */ typedef void * hwloc_backend_distances_add_handle_t; diff --git a/src/3rdparty/hwloc/include/hwloc/rename.h b/src/3rdparty/hwloc/include/hwloc/rename.h index d5687b69..7fddc600 100644 --- a/src/3rdparty/hwloc/include/hwloc/rename.h +++ b/src/3rdparty/hwloc/include/hwloc/rename.h @@ -1,6 +1,6 @@ /* * Copyright © 2009-2011 Cisco Systems, Inc. All rights reserved. - * Copyright © 2010-2022 Inria. All rights reserved. + * Copyright © 2010-2025 Inria. All rights reserved. * See COPYING in top-level directory. */ @@ -210,6 +210,7 @@ extern "C" { #define hwloc_obj_get_info_by_name HWLOC_NAME(obj_get_info_by_name) #define hwloc_obj_add_info HWLOC_NAME(obj_add_info) +#define hwloc_obj_set_subtype HWLOC_NAME(obj_set_subtype) #define HWLOC_CPUBIND_PROCESS HWLOC_NAME_CAPS(CPUBIND_PROCESS) #define HWLOC_CPUBIND_THREAD HWLOC_NAME_CAPS(CPUBIND_THREAD) @@ -232,6 +233,7 @@ extern "C" { #define HWLOC_MEMBIND_FIRSTTOUCH HWLOC_NAME_CAPS(MEMBIND_FIRSTTOUCH) #define HWLOC_MEMBIND_BIND HWLOC_NAME_CAPS(MEMBIND_BIND) #define HWLOC_MEMBIND_INTERLEAVE HWLOC_NAME_CAPS(MEMBIND_INTERLEAVE) +#define HWLOC_MEMBIND_WEIGHTED_INTERLEAVE HWLOC_NAME_CAPS(MEMBIND_WEIGHTED_INTERLEAVE) #define HWLOC_MEMBIND_NEXTTOUCH HWLOC_NAME_CAPS(MEMBIND_NEXTTOUCH) #define HWLOC_MEMBIND_MIXED HWLOC_NAME_CAPS(MEMBIND_MIXED) @@ -407,8 +409,10 @@ extern "C" { #define hwloc_local_numanode_flag_e HWLOC_NAME(local_numanode_flag_e) #define HWLOC_LOCAL_NUMANODE_FLAG_LARGER_LOCALITY HWLOC_NAME_CAPS(LOCAL_NUMANODE_FLAG_LARGER_LOCALITY) #define HWLOC_LOCAL_NUMANODE_FLAG_SMALLER_LOCALITY HWLOC_NAME_CAPS(LOCAL_NUMANODE_FLAG_SMALLER_LOCALITY) +#define HWLOC_LOCAL_NUMANODE_FLAG_INTERSECT_LOCALITY HWLOC_NAME_CAPS(LOCAL_NUMANODE_FLAG_INTERSECT_LOCALITY) #define HWLOC_LOCAL_NUMANODE_FLAG_ALL HWLOC_NAME_CAPS(LOCAL_NUMANODE_FLAG_ALL) #define hwloc_get_local_numanode_objs HWLOC_NAME(get_local_numanode_objs) +#define hwloc_topology_get_default_nodeset HWLOC_NAME(topology_get_default_nodeset) #define hwloc_memattr_get_name HWLOC_NAME(memattr_get_name) #define hwloc_memattr_get_flags HWLOC_NAME(memattr_get_flags) @@ -560,6 +564,7 @@ extern "C" { /* opencl.h */ +#define hwloc_cl_device_pci_bus_info_khr HWLOC_NAME(cl_device_pci_bus_info_khr) #define hwloc_cl_device_topology_amd HWLOC_NAME(cl_device_topology_amd) #define hwloc_opencl_get_device_pci_busid HWLOC_NAME(opencl_get_device_pci_ids) #define hwloc_opencl_get_device_cpuset HWLOC_NAME(opencl_get_device_cpuset) @@ -596,7 +601,9 @@ extern "C" { /* levelzero.h */ #define hwloc_levelzero_get_device_cpuset HWLOC_NAME(levelzero_get_device_cpuset) +#define hwloc_levelzero_get_sysman_device_cpuset HWLOC_NAME(levelzero_get_sysman_device_cpuset) #define hwloc_levelzero_get_device_osdev HWLOC_NAME(levelzero_get_device_osdev) +#define hwloc_levelzero_get_sysman_device_osdev HWLOC_NAME(levelzero_get_sysman_device_osdev) /* gl.h */ @@ -715,6 +722,8 @@ extern "C" { #define hwloc__obj_type_is_dcache HWLOC_NAME(_obj_type_is_dcache) #define hwloc__obj_type_is_icache HWLOC_NAME(_obj_type_is_icache) +#define hwloc__pci_link_speed HWLOC_NAME(_pci_link_speed) + /* private/cpuid-x86.h */ #define hwloc_have_x86_cpuid HWLOC_NAME(have_x86_cpuid) @@ -808,6 +817,8 @@ extern "C" { #define hwloc_topology_setup_defaults HWLOC_NAME(topology_setup_defaults) #define hwloc_topology_clear HWLOC_NAME(topology_clear) +#define hwloc__reconnect HWLOC_NAME(_reconnect) + #define hwloc__attach_memory_object HWLOC_NAME(insert_memory_object) #define hwloc_get_obj_by_type_and_gp_index HWLOC_NAME(get_obj_by_type_and_gp_index) diff --git a/src/3rdparty/hwloc/include/private/cpuid-x86.h b/src/3rdparty/hwloc/include/private/cpuid-x86.h index 2758afe0..1f87fdac 100644 --- a/src/3rdparty/hwloc/include/private/cpuid-x86.h +++ b/src/3rdparty/hwloc/include/private/cpuid-x86.h @@ -11,6 +11,22 @@ #ifndef HWLOC_PRIVATE_CPUID_X86_H #define HWLOC_PRIVATE_CPUID_X86_H +/* A macro for annotating memory as uninitialized when building with MSAN + * (and otherwise having no effect). See below for why this is used with + * our custom assembly. + */ +#ifdef __has_feature +#define HWLOC_HAS_FEATURE(name) __has_feature(name) +#else +#define HWLOC_HAS_FEATURE(name) 0 +#endif +#if HWLOC_HAS_FEATURE(memory_sanitizer) || defined(MEMORY_SANITIZER) +#include +#define HWLOC_ANNOTATE_MEMORY_IS_INITIALIZED(ptr, len) __msan_unpoison(ptr, len) +#else +#define HWLOC_ANNOTATE_MEMORY_IS_INITIALIZED(ptr, len) +#endif + #if (defined HWLOC_X86_32_ARCH) && (!defined HWLOC_HAVE_MSVC_CPUIDEX) static __hwloc_inline int hwloc_have_x86_cpuid(void) { @@ -71,12 +87,18 @@ static __hwloc_inline void hwloc_x86_cpuid(unsigned *eax, unsigned *ebx, unsigne "movl %k2,%1\n\t" : "+a" (*eax), "=m" (*ebx), "=&r"(sav_rbx), "+c" (*ecx), "=&d" (*edx)); + /* MSAN does not recognize the effect of the above assembly on the memory operand + * (`"=m"(*ebx)`). This may get improved in MSAN at some point in the future, e.g. + * see https://github.com/llvm/llvm-project/pull/77393. */ + HWLOC_ANNOTATE_MEMORY_IS_INITIALIZED(ebx, sizeof *ebx); #elif defined(HWLOC_X86_32_ARCH) __asm__( "mov %%ebx,%1\n\t" "cpuid\n\t" "xchg %%ebx,%1\n\t" : "+a" (*eax), "=&SD" (*ebx), "+c" (*ecx), "=&d" (*edx)); + /* See above. */ + HWLOC_ANNOTATE_MEMORY_IS_INITIALIZED(ebx, sizeof *ebx); #else #error unknown architecture #endif diff --git a/src/3rdparty/hwloc/include/private/misc.h b/src/3rdparty/hwloc/include/private/misc.h index bc57e98e..823e8a2d 100644 --- a/src/3rdparty/hwloc/include/private/misc.h +++ b/src/3rdparty/hwloc/include/private/misc.h @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2019 Inria. All rights reserved. + * Copyright © 2009-2024 Inria. All rights reserved. * Copyright © 2009-2012 Université Bordeaux * Copyright © 2011 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -186,7 +186,7 @@ hwloc_ffsl_from_ffs32(unsigned long x) /** * flsl helpers. */ -#ifdef __GNUC_____ +#ifdef __GNUC__ # if (__GNUC__ >= 4) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4)) # define hwloc_flsl(x) ((x) ? (8*sizeof(long) - __builtin_clzl(x)) : 0) @@ -573,4 +573,35 @@ typedef SSIZE_T ssize_t; # endif #endif +static __inline float +hwloc__pci_link_speed(unsigned generation, unsigned lanes) +{ + float lanespeed; + /* + * These are single-direction bandwidths only. + * + * Gen1 used NRZ with 8/10 encoding. + * PCIe Gen1 = 2.5GT/s signal-rate per lane x 8/10 = 0.25GB/s data-rate per lane + * PCIe Gen2 = 5 GT/s signal-rate per lane x 8/10 = 0.5 GB/s data-rate per lane + * Gen3 switched to NRZ with 128/130 encoding. + * PCIe Gen3 = 8 GT/s signal-rate per lane x 128/130 = 1 GB/s data-rate per lane + * PCIe Gen4 = 16 GT/s signal-rate per lane x 128/130 = 2 GB/s data-rate per lane + * PCIe Gen5 = 32 GT/s signal-rate per lane x 128/130 = 4 GB/s data-rate per lane + * Gen6 switched to PAM with with 242/256 FLIT (242B payload protected by 8B CRC + 6B FEC). + * PCIe Gen6 = 64 GT/s signal-rate per lane x 242/256 = 8 GB/s data-rate per lane + * PCIe Gen7 = 128GT/s signal-rate per lane x 242/256 = 16 GB/s data-rate per lane + */ + + /* lanespeed in Gbit/s */ + if (generation <= 2) + lanespeed = 2.5f * generation * 0.8f; + else if (generation <= 5) + lanespeed = 8.0f * (1<<(generation-3)) * 128/130; + else + lanespeed = 8.0f * (1<<(generation-3)) * 242/256; /* assume Gen8 will be 256 GT/s and so on */ + + /* linkspeed in GB/s */ + return lanespeed * lanes / 8; +} + #endif /* HWLOC_PRIVATE_MISC_H */ diff --git a/src/3rdparty/hwloc/include/private/private.h b/src/3rdparty/hwloc/include/private/private.h index 3e3f71d9..0c262aa6 100644 --- a/src/3rdparty/hwloc/include/private/private.h +++ b/src/3rdparty/hwloc/include/private/private.h @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2023 Inria. All rights reserved. + * Copyright © 2009-2025 Inria. All rights reserved. * Copyright © 2009-2012, 2020 Université Bordeaux * Copyright © 2009-2011 Cisco Systems, Inc. All rights reserved. * @@ -302,6 +302,9 @@ extern void hwloc__reorder_children(hwloc_obj_t parent); extern void hwloc_topology_setup_defaults(struct hwloc_topology *topology); extern void hwloc_topology_clear(struct hwloc_topology *topology); +#define _HWLOC_RECONNECT_FLAG_KEEPSTRUCTURE (1UL<<0) +extern int hwloc__reconnect(struct hwloc_topology *topology, unsigned long flags); + /* insert memory object as memory child of normal parent */ extern struct hwloc_obj * hwloc__attach_memory_object(struct hwloc_topology *topology, hwloc_obj_t parent, hwloc_obj_t obj, const char *reason); diff --git a/src/3rdparty/hwloc/src/bind.c b/src/3rdparty/hwloc/src/bind.c index 2b5d0994..580580e7 100644 --- a/src/3rdparty/hwloc/src/bind.c +++ b/src/3rdparty/hwloc/src/bind.c @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2020 Inria. All rights reserved. + * Copyright © 2009-2024 Inria. All rights reserved. * Copyright © 2009-2010, 2012 Université Bordeaux * Copyright © 2011-2015 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -287,6 +287,7 @@ static __hwloc_inline int hwloc__check_membind_policy(hwloc_membind_policy_t pol || policy == HWLOC_MEMBIND_FIRSTTOUCH || policy == HWLOC_MEMBIND_BIND || policy == HWLOC_MEMBIND_INTERLEAVE + || policy == HWLOC_MEMBIND_WEIGHTED_INTERLEAVE || policy == HWLOC_MEMBIND_NEXTTOUCH) return 0; return -1; diff --git a/src/3rdparty/hwloc/src/bitmap.c b/src/3rdparty/hwloc/src/bitmap.c index cf071edb..b7825b46 100644 --- a/src/3rdparty/hwloc/src/bitmap.c +++ b/src/3rdparty/hwloc/src/bitmap.c @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2020 Inria. All rights reserved. + * Copyright © 2009-2024 Inria. All rights reserved. * Copyright © 2009-2011 Université Bordeaux * Copyright © 2009-2011 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -245,6 +245,7 @@ int hwloc_bitmap_copy(struct hwloc_bitmap_s * dst, const struct hwloc_bitmap_s * /* Strings always use 32bit groups */ #define HWLOC_PRIxSUBBITMAP "%08lx" #define HWLOC_BITMAP_SUBSTRING_SIZE 32 +#define HWLOC_BITMAP_SUBSTRING_FULL_VALUE 0xFFFFFFFFUL #define HWLOC_BITMAP_SUBSTRING_LENGTH (HWLOC_BITMAP_SUBSTRING_SIZE/4) #define HWLOC_BITMAP_STRING_PER_LONG (HWLOC_BITS_PER_LONG/HWLOC_BITMAP_SUBSTRING_SIZE) @@ -261,6 +262,7 @@ int hwloc_bitmap_snprintf(char * __hwloc_restrict buf, size_t buflen, const stru const unsigned long accum_mask = ~0UL; #else /* HWLOC_BITS_PER_LONG != HWLOC_BITMAP_SUBSTRING_SIZE */ const unsigned long accum_mask = ((1UL << HWLOC_BITMAP_SUBSTRING_SIZE) - 1) << (HWLOC_BITS_PER_LONG - HWLOC_BITMAP_SUBSTRING_SIZE); + int merge_with_infinite_prefix = 0; #endif /* HWLOC_BITS_PER_LONG != HWLOC_BITMAP_SUBSTRING_SIZE */ HWLOC__BITMAP_CHECK(set); @@ -279,6 +281,9 @@ int hwloc_bitmap_snprintf(char * __hwloc_restrict buf, size_t buflen, const stru res = size>0 ? (int)size - 1 : 0; tmp += res; size -= res; +#if HWLOC_BITS_PER_LONG > HWLOC_BITMAP_SUBSTRING_SIZE + merge_with_infinite_prefix = 1; +#endif } i=(int) set->ulongs_count-1; @@ -294,16 +299,24 @@ int hwloc_bitmap_snprintf(char * __hwloc_restrict buf, size_t buflen, const stru } while (i>=0 || accumed) { + unsigned long value; + /* Refill accumulator */ if (!accumed) { accum = set->ulongs[i--]; accumed = HWLOC_BITS_PER_LONG; } + value = (accum & accum_mask) >> (HWLOC_BITS_PER_LONG - HWLOC_BITMAP_SUBSTRING_SIZE); - if (accum & accum_mask) { +#if HWLOC_BITS_PER_LONG > HWLOC_BITMAP_SUBSTRING_SIZE + if (merge_with_infinite_prefix && value == HWLOC_BITMAP_SUBSTRING_FULL_VALUE) { + /* first full subbitmap merged with infinite prefix */ + res = 0; + } else +#endif + if (value) { /* print the whole subset if not empty */ - res = hwloc_snprintf(tmp, size, needcomma ? ",0x" HWLOC_PRIxSUBBITMAP : "0x" HWLOC_PRIxSUBBITMAP, - (accum & accum_mask) >> (HWLOC_BITS_PER_LONG - HWLOC_BITMAP_SUBSTRING_SIZE)); + res = hwloc_snprintf(tmp, size, needcomma ? ",0x" HWLOC_PRIxSUBBITMAP : "0x" HWLOC_PRIxSUBBITMAP, value); needcomma = 1; } else if (i == -1 && accumed == HWLOC_BITMAP_SUBSTRING_SIZE) { /* print a single 0 to mark the last subset */ @@ -323,6 +336,7 @@ int hwloc_bitmap_snprintf(char * __hwloc_restrict buf, size_t buflen, const stru #else accum <<= HWLOC_BITMAP_SUBSTRING_SIZE; accumed -= HWLOC_BITMAP_SUBSTRING_SIZE; + merge_with_infinite_prefix = 0; #endif if (res >= size) @@ -362,7 +376,8 @@ int hwloc_bitmap_sscanf(struct hwloc_bitmap_s *set, const char * __hwloc_restric { const char * current = string; unsigned long accum = 0; - int count=0; + int count = 0; + int ulongcount; int infinite = 0; /* count how many substrings there are */ @@ -383,9 +398,20 @@ int hwloc_bitmap_sscanf(struct hwloc_bitmap_s *set, const char * __hwloc_restric count--; } - if (hwloc_bitmap_reset_by_ulongs(set, (count + HWLOC_BITMAP_STRING_PER_LONG - 1) / HWLOC_BITMAP_STRING_PER_LONG) < 0) + ulongcount = (count + HWLOC_BITMAP_STRING_PER_LONG - 1) / HWLOC_BITMAP_STRING_PER_LONG; + if (hwloc_bitmap_reset_by_ulongs(set, ulongcount) < 0) return -1; - set->infinite = 0; + + set->infinite = 0; /* will be updated later */ + +#if HWLOC_BITS_PER_LONG != HWLOC_BITMAP_SUBSTRING_SIZE + if (infinite && (count % HWLOC_BITMAP_STRING_PER_LONG) != 0) { + /* accumulate substrings of the first ulong that are hidden in the infinite prefix */ + int i; + for(i = (count % HWLOC_BITMAP_STRING_PER_LONG); i < HWLOC_BITMAP_STRING_PER_LONG; i++) + accum |= (HWLOC_BITMAP_SUBSTRING_FULL_VALUE << (i*HWLOC_BITMAP_SUBSTRING_SIZE)); + } +#endif while (*current != '\0') { unsigned long val; @@ -544,6 +570,9 @@ int hwloc_bitmap_taskset_snprintf(char * __hwloc_restrict buf, size_t buflen, co ssize_t size = buflen; char *tmp = buf; int res, ret = 0; +#if HWLOC_BITS_PER_LONG == 64 + int merge_with_infinite_prefix = 0; +#endif int started = 0; int i; @@ -563,6 +592,9 @@ int hwloc_bitmap_taskset_snprintf(char * __hwloc_restrict buf, size_t buflen, co res = size>0 ? (int)size - 1 : 0; tmp += res; size -= res; +#if HWLOC_BITS_PER_LONG == 64 + merge_with_infinite_prefix = 1; +#endif } i=set->ulongs_count-1; @@ -582,7 +614,11 @@ int hwloc_bitmap_taskset_snprintf(char * __hwloc_restrict buf, size_t buflen, co if (started) { /* print the whole subset */ #if HWLOC_BITS_PER_LONG == 64 - res = hwloc_snprintf(tmp, size, "%016lx", val); + if (merge_with_infinite_prefix && (val & 0xffffffff00000000UL) == 0xffffffff00000000UL) { + res = hwloc_snprintf(tmp, size, "%08lx", val & 0xffffffffUL); + } else { + res = hwloc_snprintf(tmp, size, "%016lx", val); + } #else res = hwloc_snprintf(tmp, size, "%08lx", val); #endif @@ -599,6 +635,9 @@ int hwloc_bitmap_taskset_snprintf(char * __hwloc_restrict buf, size_t buflen, co res = size>0 ? (int)size - 1 : 0; tmp += res; size -= res; +#if HWLOC_BITS_PER_LONG == 64 + merge_with_infinite_prefix = 0; +#endif } /* if didn't display anything, display 0x0 */ @@ -679,6 +718,10 @@ int hwloc_bitmap_taskset_sscanf(struct hwloc_bitmap_s *set, const char * __hwloc goto failed; set->ulongs[count-1] = val; + if (infinite && tmpchars != HWLOC_BITS_PER_LONG/4) { + /* infinite prefix with partial substring, fill remaining bits */ + set->ulongs[count-1] |= (~0ULL)<<(4*tmpchars); + } current += tmpchars; chars -= tmpchars; diff --git a/src/3rdparty/hwloc/src/cpukinds.c b/src/3rdparty/hwloc/src/cpukinds.c index 6c7c087f..321b12d4 100644 --- a/src/3rdparty/hwloc/src/cpukinds.c +++ b/src/3rdparty/hwloc/src/cpukinds.c @@ -1,5 +1,5 @@ /* - * Copyright © 2020-2022 Inria. All rights reserved. + * Copyright © 2020-2024 Inria. All rights reserved. * See COPYING in top-level directory. */ @@ -50,6 +50,7 @@ hwloc_internal_cpukinds_dup(hwloc_topology_t new, hwloc_topology_t old) return -1; new->cpukinds = kinds; new->nr_cpukinds = old->nr_cpukinds; + new->nr_cpukinds_allocated = old->nr_cpukinds; memcpy(kinds, old->cpukinds, old->nr_cpukinds * sizeof(*kinds)); for(i=0;inr_cpukinds; i++) { diff --git a/src/3rdparty/hwloc/src/distances.c b/src/3rdparty/hwloc/src/distances.c index bfc7d61d..28c586c7 100644 --- a/src/3rdparty/hwloc/src/distances.c +++ b/src/3rdparty/hwloc/src/distances.c @@ -1,5 +1,5 @@ /* - * Copyright © 2010-2022 Inria. All rights reserved. + * Copyright © 2010-2025 Inria. All rights reserved. * Copyright © 2011-2012 Université Bordeaux * Copyright © 2011 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -624,8 +624,8 @@ void * hwloc_distances_add_create(hwloc_topology_t topology, return NULL; } if ((kind & ~HWLOC_DISTANCES_KIND_ALL) - || hwloc_weight_long(kind & HWLOC_DISTANCES_KIND_FROM_ALL) != 1 - || hwloc_weight_long(kind & HWLOC_DISTANCES_KIND_MEANS_ALL) != 1) { + || hwloc_weight_long(kind & HWLOC_DISTANCES_KIND_FROM_ALL) > 1 + || hwloc_weight_long(kind & HWLOC_DISTANCES_KIND_MEANS_ALL) > 1) { errno = EINVAL; return NULL; } @@ -699,7 +699,7 @@ hwloc_distances_add_commit(hwloc_topology_t topology, } /* in case we added some groups, see if we need to reconnect */ - hwloc_topology_reconnect(topology, 0); + hwloc__reconnect(topology, 0); return 0; @@ -1387,19 +1387,12 @@ static __hwloc_inline int is_nvswitch(hwloc_obj_t obj) } static int -hwloc__distances_transform_merge_switch_ports(hwloc_topology_t topology, - struct hwloc_distances_s *distances) +hwloc__distances_transform_merge_switch_ports(struct hwloc_distances_s *distances) { - struct hwloc_internal_distances_s *dist = hwloc__internal_distances_from_public(topology, distances); hwloc_obj_t *objs = distances->objs; hwloc_uint64_t *values = distances->values; unsigned first, i, j, nbobjs = distances->nbobjs; - if (strcmp(dist->name, "NVLinkBandwidth")) { - errno = EINVAL; - return -1; - } - /* find the first port */ first = (unsigned) -1; for(i=0; iobjs; hwloc_uint64_t *values = distances->values; unsigned nbobjs = distances->nbobjs; unsigned i, j, k; - if (strcmp(dist->name, "NVLinkBandwidth")) { - errno = EINVAL; - return -1; - } - for(i=0; i bw_sw2j ? bw_sw2j : bw_i2sw; + /* bandwidth from i to j now gets indirect bandwidth too, min(i2sw,sw2j) */ + values[i*nbobjs+j] += bw_i2sw > bw_sw2j ? bw_sw2j : bw_i2sw; } } @@ -1476,7 +1462,7 @@ hwloc__distances_transform_transitive_closure(hwloc_topology_t topology, } int -hwloc_distances_transform(hwloc_topology_t topology, +hwloc_distances_transform(hwloc_topology_t topology __hwloc_attribute_unused, struct hwloc_distances_s *distances, enum hwloc_distances_transform_e transform, void *transform_attr, @@ -1495,13 +1481,13 @@ hwloc_distances_transform(hwloc_topology_t topology, case HWLOC_DISTANCES_TRANSFORM_MERGE_SWITCH_PORTS: { int err; - err = hwloc__distances_transform_merge_switch_ports(topology, distances); + err = hwloc__distances_transform_merge_switch_ports(distances); if (!err) err = hwloc__distances_transform_remove_null(distances); return err; } case HWLOC_DISTANCES_TRANSFORM_TRANSITIVE_CLOSURE: - return hwloc__distances_transform_transitive_closure(topology, distances); + return hwloc__distances_transform_transitive_closure(distances); default: errno = EINVAL; return -1; diff --git a/src/3rdparty/hwloc/src/memattrs.c b/src/3rdparty/hwloc/src/memattrs.c index ab945471..f2c27565 100644 --- a/src/3rdparty/hwloc/src/memattrs.c +++ b/src/3rdparty/hwloc/src/memattrs.c @@ -1,5 +1,5 @@ /* - * Copyright © 2020-2023 Inria. All rights reserved. + * Copyright © 2020-2025 Inria. All rights reserved. * See COPYING in top-level directory. */ @@ -14,13 +14,26 @@ */ static __hwloc_inline -hwloc_uint64_t hwloc__memattr_get_convenience_value(hwloc_memattr_id_t id, - hwloc_obj_t node) +int hwloc__memattr_get_convenience_value(hwloc_memattr_id_t id, + hwloc_obj_t node, + hwloc_uint64_t *valuep) { - if (id == HWLOC_MEMATTR_ID_CAPACITY) - return node->attr->numanode.local_memory; - else if (id == HWLOC_MEMATTR_ID_LOCALITY) - return hwloc_bitmap_weight(node->cpuset); + if (id == HWLOC_MEMATTR_ID_CAPACITY) { + if (node->type != HWLOC_OBJ_NUMANODE) { + errno = EINVAL; + return -1; + } + *valuep = node->attr->numanode.local_memory; + return 0; + } + else if (id == HWLOC_MEMATTR_ID_LOCALITY) { + if (!node->cpuset) { + errno = EINVAL; + return -1; + } + *valuep = hwloc_bitmap_weight(node->cpuset); + return 0; + } else assert(0); return 0; /* shut up the compiler */ @@ -622,7 +635,7 @@ hwloc_memattr_get_targets(hwloc_topology_t topology, if (foundiflags & HWLOC_IMATTR_FLAG_CONVENIENCE) { /* convenience attributes */ - *valuep = hwloc__memattr_get_convenience_value(id, target_node); - return 0; + return hwloc__memattr_get_convenience_value(id, target_node, valuep); } /* normal attributes */ @@ -936,7 +948,7 @@ hwloc_memattr_set_value(hwloc_topology_t topology, { struct hwloc_internal_location_s iloc, *ilocp; - if (flags) { + if (flags || !target_node) { errno = EINVAL; return -1; } @@ -1007,10 +1019,10 @@ hwloc_memattr_get_best_target(hwloc_topology_t topology, /* convenience attributes */ for(j=0; ; j++) { hwloc_obj_t node = hwloc_get_obj_by_type(topology, HWLOC_OBJ_NUMANODE, j); - hwloc_uint64_t value; + hwloc_uint64_t value = 0; if (!node) break; - value = hwloc__memattr_get_convenience_value(id, node); + hwloc__memattr_get_convenience_value(id, node, &value); hwloc__update_best_target(&best, &best_value, &found, node, value, imattr->flags & HWLOC_MEMATTR_FLAG_HIGHER_FIRST); @@ -1093,7 +1105,7 @@ hwloc_memattr_get_best_initiator(hwloc_topology_t topology, int found; unsigned i; - if (flags) { + if (flags || !target_node) { errno = EINVAL; return -1; } @@ -1146,6 +1158,8 @@ match_local_obj_cpuset(hwloc_obj_t node, hwloc_cpuset_t cpuset, unsigned long fl { if (flags & HWLOC_LOCAL_NUMANODE_FLAG_ALL) return 1; + if (flags & HWLOC_LOCAL_NUMANODE_FLAG_INTERSECT_LOCALITY) + return hwloc_bitmap_intersects(node->cpuset, cpuset); if ((flags & HWLOC_LOCAL_NUMANODE_FLAG_LARGER_LOCALITY) && hwloc_bitmap_isincluded(cpuset, node->cpuset)) return 1; @@ -1168,6 +1182,7 @@ hwloc_get_local_numanode_objs(hwloc_topology_t topology, if (flags & ~(HWLOC_LOCAL_NUMANODE_FLAG_SMALLER_LOCALITY |HWLOC_LOCAL_NUMANODE_FLAG_LARGER_LOCALITY + |HWLOC_LOCAL_NUMANODE_FLAG_INTERSECT_LOCALITY | HWLOC_LOCAL_NUMANODE_FLAG_ALL)) { errno = EINVAL; return -1; @@ -1214,6 +1229,93 @@ hwloc_get_local_numanode_objs(hwloc_topology_t topology, return 0; } +static int compare_nodes_by_os_index(const void *_a, const void *_b) +{ + const hwloc_obj_t * a = _a, * b = _b; + return (*a)->os_index - (*b)->os_index; +} + +int +hwloc_topology_get_default_nodeset(hwloc_topology_t topology, + hwloc_nodeset_t nodeset, + unsigned long flags) +{ + hwloc_obj_t *nodes; + hwloc_bitmap_t remainingcpuset; + unsigned nrnodes, i; + const char *first_subtype; + + if (flags) { + errno = EINVAL; + goto out; + } + + remainingcpuset = hwloc_bitmap_dup(topology->levels[0][0]->cpuset); + if (!remainingcpuset) + goto out; + + nrnodes = topology->slevels[HWLOC_SLEVEL_NUMANODE].nbobjs; + nodes = malloc(nrnodes * sizeof(*nodes)); + if (!nodes) + goto out_with_remainingcpuset; + + memcpy(nodes, topology->slevels[HWLOC_SLEVEL_NUMANODE].objs, nrnodes * sizeof(*nodes)); + qsort(nodes, nrnodes, sizeof(*nodes), compare_nodes_by_os_index); + + hwloc_bitmap_zero(nodeset); + + /* always take the first node (FIXME: except if unexpected subtype?) */ + first_subtype = nodes[0]->subtype; + hwloc_bitmap_set(nodeset, nodes[0]->os_index); + hwloc_bitmap_andnot(remainingcpuset, remainingcpuset, nodes[0]->cpuset); + + /* use all non-intersecting nodes with same subtype */ + for(i=1; isubtype || strcmp(first_subtype, nodes[i]->subtype)) + continue; + } else if (nodes[i]->subtype) { + continue; + } + /* take non-overlapping nodes */ + if (hwloc_bitmap_isincluded(nodes[i]->cpuset, remainingcpuset) /* can be empty */) { + hwloc_bitmap_set(nodeset, nodes[i]->os_index); + hwloc_bitmap_andnot(remainingcpuset, remainingcpuset, nodes[i]->cpuset); + } + /* more needed? */ + if (hwloc_bitmap_iszero(remainingcpuset)) + goto done; + } + + /* find more nodes to cover the entire topology cpuset. + * only take what's necessary: first nodes, non-empty */ + for(i=1; icpuset, remainingcpuset) + && !hwloc_bitmap_iszero(nodes[i]->cpuset)) { + hwloc_bitmap_set(nodeset, nodes[i]->os_index); + hwloc_bitmap_andnot(remainingcpuset, remainingcpuset, nodes[i]->cpuset); + } + /* more needed? */ + if (hwloc_bitmap_iszero(remainingcpuset)) + goto done; + } + + done: + free(nodes); + hwloc_bitmap_free(remainingcpuset); + return 0; + + out_with_remainingcpuset: + hwloc_bitmap_free(remainingcpuset); + out: + return -1; +} + /************************************** * Using memattrs to identify HBM/DRAM @@ -1421,10 +1523,15 @@ hwloc__group_memory_tiers(hwloc_topology_t topology, } } - /* Sort nodes. - * We could also sort by the existing subtype. - * KNL is the only case where subtypes are set in backends, but we set memattrs as well there. - * Also HWLOC_MEMTIERS_REFRESH would be a special value to ignore existing subtypes. + /* Sort nodes by tier type and bandwidth. + * + * We could also use the existing subtype but it's not clear it'd be better. + * For NVIDIA GPU, "GPUMemory" is set in the Linux backend, and used above to set tier type anyway. + * For KNL, the Linux backend sets subtypes and memattrs, sorting by memattrs already works fine. + * Existing subtypes could have been imported from XML, usually mostly OK except maybe SPM (fallback for I don't know)? + * An envvar (or HWLOC_MEMTIERS_REFRESH special value?) could be passed to ignore existing subtypes, + * but "GPUMemory" wouldn't be available anymore, we'd have to use something else like "PCIBusId", + * but that one might not always be specific to GPU-backed NUMA nodes? */ hwloc_debug("Sorting memory node infos...\n"); qsort(nodeinfos, n, sizeof(*nodeinfos), compare_node_infos_by_type_and_bw); @@ -1806,6 +1913,12 @@ hwloc__apply_memory_tiers_subtypes(hwloc_topology_t topology, } } } + if (nr_tiers > 1) { + hwloc_obj_t root = hwloc_get_root_obj(topology); + char tmp[20]; + snprintf(tmp, sizeof(tmp), "%u", nr_tiers); + hwloc__add_info_nodup(&root->infos, &root->infos_count, "MemoryTiersNr", tmp, 1); + } } int diff --git a/src/3rdparty/hwloc/src/pci-common.c b/src/3rdparty/hwloc/src/pci-common.c index b5a4b544..feb1834d 100644 --- a/src/3rdparty/hwloc/src/pci-common.c +++ b/src/3rdparty/hwloc/src/pci-common.c @@ -1,5 +1,5 @@ /* - * Copyright © 2009-2022 Inria. All rights reserved. + * Copyright © 2009-2024 Inria. All rights reserved. * See COPYING in top-level directory. */ @@ -886,36 +886,12 @@ hwloc_pcidisc_find_linkspeed(const unsigned char *config, unsigned offset, float *linkspeed) { unsigned linksta, speed, width; - float lanespeed; memcpy(&linksta, &config[offset + HWLOC_PCI_EXP_LNKSTA], 4); speed = linksta & HWLOC_PCI_EXP_LNKSTA_SPEED; /* PCIe generation */ width = (linksta & HWLOC_PCI_EXP_LNKSTA_WIDTH) >> 4; /* how many lanes */ - /* - * These are single-direction bandwidths only. - * - * Gen1 used NRZ with 8/10 encoding. - * PCIe Gen1 = 2.5GT/s signal-rate per lane x 8/10 = 0.25GB/s data-rate per lane - * PCIe Gen2 = 5 GT/s signal-rate per lane x 8/10 = 0.5 GB/s data-rate per lane - * Gen3 switched to NRZ with 128/130 encoding. - * PCIe Gen3 = 8 GT/s signal-rate per lane x 128/130 = 1 GB/s data-rate per lane - * PCIe Gen4 = 16 GT/s signal-rate per lane x 128/130 = 2 GB/s data-rate per lane - * PCIe Gen5 = 32 GT/s signal-rate per lane x 128/130 = 4 GB/s data-rate per lane - * Gen6 switched to PAM with with 242/256 FLIT (242B payload protected by 8B CRC + 6B FEC). - * PCIe Gen6 = 64 GT/s signal-rate per lane x 242/256 = 8 GB/s data-rate per lane - * PCIe Gen7 = 128GT/s signal-rate per lane x 242/256 = 16 GB/s data-rate per lane - */ - /* lanespeed in Gbit/s */ - if (speed <= 2) - lanespeed = 2.5f * speed * 0.8f; - else if (speed <= 5) - lanespeed = 8.0f * (1<<(speed-3)) * 128/130; - else - lanespeed = 8.0f * (1<<(speed-3)) * 242/256; /* assume Gen8 will be 256 GT/s and so on */ - - /* linkspeed in GB/s */ - *linkspeed = lanespeed * width / 8; + *linkspeed = hwloc__pci_link_speed(speed, width); return 0; } diff --git a/src/3rdparty/hwloc/src/topology-windows.c b/src/3rdparty/hwloc/src/topology-windows.c index e187bb12..8aa19aeb 100644 --- a/src/3rdparty/hwloc/src/topology-windows.c +++ b/src/3rdparty/hwloc/src/topology-windows.c @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2023 Inria. All rights reserved. + * Copyright © 2009-2025 Inria. All rights reserved. * Copyright © 2009-2012, 2020 Université Bordeaux * Copyright © 2011 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -56,6 +56,9 @@ typedef enum _LOGICAL_PROCESSOR_RELATIONSHIP { RelationCache, RelationProcessorPackage, RelationGroup, + RelationProcessorDie, + RelationNumaNodeEx, /* only used to *request* extended numa info only, but included in RelationAll, never returned on output */ + RelationProcessorModule, RelationAll = 0xffff } LOGICAL_PROCESSOR_RELATIONSHIP; #else /* HAVE_LOGICAL_PROCESSOR_RELATIONSHIP */ @@ -64,6 +67,11 @@ typedef enum _LOGICAL_PROCESSOR_RELATIONSHIP { # define RelationGroup 4 # define RelationAll 0xffff # endif /* HAVE_RELATIONPROCESSORPACKAGE */ +# ifndef HAVE_RELATIONPROCESSORDIE +# define RelationProcessorDie 5 +# define RelationNumaNodeEx 6 +# define RelationProcessorModule 7 +# endif #endif /* HAVE_LOGICAL_PROCESSOR_RELATIONSHIP */ #ifndef HAVE_GROUP_AFFINITY @@ -220,7 +228,7 @@ static void hwloc_win_get_function_ptrs(void) #pragma GCC diagnostic ignored "-Wcast-function-type" #endif - kernel32 = LoadLibrary("kernel32.dll"); + kernel32 = LoadLibrary(TEXT("kernel32.dll")); if (kernel32) { GetActiveProcessorGroupCountProc = (PFN_GETACTIVEPROCESSORGROUPCOUNT) GetProcAddress(kernel32, "GetActiveProcessorGroupCount"); @@ -249,12 +257,12 @@ static void hwloc_win_get_function_ptrs(void) } if (!QueryWorkingSetExProc) { - HMODULE psapi = LoadLibrary("psapi.dll"); + HMODULE psapi = LoadLibrary(TEXT("psapi.dll")); if (psapi) QueryWorkingSetExProc = (PFN_QUERYWORKINGSETEX) GetProcAddress(psapi, "QueryWorkingSetEx"); } - ntdll = GetModuleHandle("ntdll"); + ntdll = GetModuleHandle(TEXT("ntdll")); RtlGetVersionProc = (PFN_RTLGETVERSION) GetProcAddress(ntdll, "RtlGetVersion"); #if HWLOC_HAVE_GCC_W_CAST_FUNCTION_TYPE @@ -366,7 +374,7 @@ hwloc_win_get_processor_groups(void) hwloc_debug("found %lu windows processor groups\n", nr_processor_groups); if (nr_processor_groups > 1 && SIZEOF_VOID_P == 4) { - if (HWLOC_SHOW_ALL_ERRORS()) + if (HWLOC_SHOW_CRITICAL_ERRORS()) fprintf(stderr, "hwloc/windows: multiple processor groups found on 32bits Windows, topology may be invalid/incomplete.\n"); } @@ -1068,6 +1076,7 @@ hwloc_look_windows(struct hwloc_backend *backend, struct hwloc_disc_status *dsta id = HWLOC_UNKNOWN_INDEX; switch (procInfo->Relationship) { + case RelationNumaNodeEx: /* only used on input anyway */ case RelationNumaNode: type = HWLOC_OBJ_NUMANODE; /* Starting with Windows 11 and Server 2022, the GroupCount field is valid and >=1 @@ -1087,9 +1096,19 @@ hwloc_look_windows(struct hwloc_backend *backend, struct hwloc_disc_status *dsta break; case RelationProcessorPackage: type = HWLOC_OBJ_PACKAGE; + num = procInfo->Processor.GroupCount; + GroupMask = procInfo->Processor.GroupMask; + break; + case RelationProcessorDie: + type = HWLOC_OBJ_DIE; num = procInfo->Processor.GroupCount; GroupMask = procInfo->Processor.GroupMask; - break; + break; + case RelationProcessorModule: + type = HWLOC_OBJ_GROUP; + num = procInfo->Processor.GroupCount; + GroupMask = procInfo->Processor.GroupMask; + break; case RelationCache: type = (procInfo->Cache.Type == CacheInstruction ? HWLOC_OBJ_L1ICACHE : HWLOC_OBJ_L1CACHE) + procInfo->Cache.Level - 1; /* GroupCount added approximately with NumaNode.GroupCount above */ @@ -1211,6 +1230,19 @@ hwloc_look_windows(struct hwloc_backend *backend, struct hwloc_disc_status *dsta continue; } break; + case HWLOC_OBJ_GROUP: + switch (procInfo->Relationship) { + case RelationGroup: + obj->attr->group.kind = HWLOC_GROUP_KIND_WINDOWS_PROCESSOR_GROUP; + break; + case RelationProcessorModule: + obj->attr->group.kind = HWLOC_GROUP_KIND_INTEL_MODULE; + obj->subtype = strdup("Module"); + break; + default: + obj->attr->group.kind = HWLOC_GROUP_KIND_WINDOWS_RELATIONSHIP_UNKNOWN; + } + break; default: break; } diff --git a/src/3rdparty/hwloc/src/topology-x86.c b/src/3rdparty/hwloc/src/topology-x86.c index 7aabd168..42248f68 100644 --- a/src/3rdparty/hwloc/src/topology-x86.c +++ b/src/3rdparty/hwloc/src/topology-x86.c @@ -1,11 +1,11 @@ /* - * Copyright © 2010-2023 Inria. All rights reserved. + * Copyright © 2010-2025 Inria. All rights reserved. * Copyright © 2010-2013 Université Bordeaux * Copyright © 2010-2011 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. * * - * This backend is only used when the operating system does not export + * This backend is mostly used when the operating system does not export * the necessary hardware topology information to user-space applications. * Currently, FreeBSD and NetBSD only add PUs and then fallback to this * backend for CPU/Cache discovery. @@ -15,6 +15,7 @@ * on various architectures, without having to use this x86-specific code. * But this backend is still used after them to annotate some objects with * additional details (CPU info in Package, Inclusiveness in Caches). + * It may also be enabled manually to work-around bugs in native OS discovery. */ #include "private/autogen/config.h" @@ -487,7 +488,7 @@ static void read_amd_cores_legacy(struct procinfo *infos, struct cpuiddump *src_ } /* AMD unit/node from CPUID 0x8000001e leaf (topoext) */ -static void read_amd_cores_topoext(struct hwloc_x86_backend_data_s *data, struct procinfo *infos, unsigned long flags, struct cpuiddump *src_cpuiddump) +static void read_amd_cores_topoext(struct hwloc_x86_backend_data_s *data, struct procinfo *infos, unsigned long flags __hwloc_attribute_unused, struct cpuiddump *src_cpuiddump) { unsigned apic_id, nodes_per_proc = 0; unsigned eax, ebx, ecx, edx; @@ -496,7 +497,6 @@ static void read_amd_cores_topoext(struct hwloc_x86_backend_data_s *data, struct cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump); infos->apicid = apic_id = eax; - if (flags & HWLOC_X86_DISC_FLAG_TOPOEXT_NUMANODES) { if (infos->cpufamilynumber == 0x16) { /* ecx is reserved */ infos->ids[NODE] = 0; @@ -511,7 +511,6 @@ static void read_amd_cores_topoext(struct hwloc_x86_backend_data_s *data, struct || (infos->cpufamilynumber == 0x19 && nodes_per_proc > 1)) { hwloc_debug("warning: undefined nodes_per_proc value %u, assuming it means %u\n", nodes_per_proc, nodes_per_proc); } - } if (infos->cpufamilynumber <= 0x16) { /* topoext appeared in 0x15 and compute-units were only used in 0x15 and 0x16 */ unsigned cores_per_unit; @@ -533,9 +532,9 @@ static void read_amd_cores_topoext(struct hwloc_x86_backend_data_s *data, struct } /* Intel core/thread or even die/module/tile from CPUID 0x0b or 0x1f leaves (v1 and v2 extended topology enumeration) - * or AMD complex/ccd from CPUID 0x80000026 (extended CPU topology) + * or AMD core/thread or even complex/ccd from CPUID 0x0b or 0x80000026 (extended CPU topology) */ -static void read_extended_topo(struct hwloc_x86_backend_data_s *data, struct procinfo *infos, unsigned leaf, enum cpuid_type cpuid_type, struct cpuiddump *src_cpuiddump) +static void read_extended_topo(struct hwloc_x86_backend_data_s *data, struct procinfo *infos, unsigned leaf, enum cpuid_type cpuid_type __hwloc_attribute_unused, struct cpuiddump *src_cpuiddump) { unsigned level, apic_nextshift, apic_type, apic_id = 0, apic_shift = 0, id; unsigned threadid __hwloc_attribute_unused = 0; /* shut-up compiler */ @@ -547,20 +546,15 @@ static void read_extended_topo(struct hwloc_x86_backend_data_s *data, struct pro eax = leaf; cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump); /* Intel specifies that the 0x0b/0x1f loop should stop when we get "invalid domain" (0 in ecx[8:15]) - * (if so, we also get 0 in eax/ebx for invalid subleaves). + * (if so, we also get 0 in eax/ebx for invalid subleaves). Zhaoxin implements this too. * However AMD rather says that the 0x80000026/0x0b loop should stop when we get "no thread at this level" (0 in ebx[0:15]). - * Zhaoxin follows the Intel specs but also returns "no thread at this level" for the last *valid* level (at least on KH-4000). - * From the Linux kernel code, it's very likely that AMD also returns "invalid domain" - * (because detect_extended_topology() uses that for all x86 CPUs) - * but keep with the official doc until AMD can clarify that (see #593). + * + * Linux kernel <= 6.8 used "invalid domain" for both Intel and AMD (in detect_extended_topology()) + * but x86 discovery revamp in 6.9 now properly checks both Intel and AMD conditions (in topo_subleaf()). + * So let's assume we are allowed to break-out once one of the Intel+AMD conditions is met. */ - if (cpuid_type == amd) { - if (!(ebx & 0xffff)) - break; - } else { - if (!(ecx & 0xff00)) - break; - } + if (!(ebx & 0xffff) || !(ecx & 0xff00)) + break; apic_packageshift = eax & 0x1f; } @@ -572,13 +566,8 @@ static void read_extended_topo(struct hwloc_x86_backend_data_s *data, struct pro ecx = level; eax = leaf; cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump); - if (cpuid_type == amd) { - if (!(ebx & 0xffff)) - break; - } else { - if (!(ecx & 0xff00)) - break; - } + if (!(ebx & 0xffff) || !(ecx & 0xff00)) + break; apic_nextshift = eax & 0x1f; apic_type = (ecx & 0xff00) >> 8; apic_id = edx; @@ -664,7 +653,13 @@ static void look_proc(struct hwloc_backend *backend, struct procinfo *infos, uns cpuid_or_from_dump(&eax, &ebx, &ecx, &edx, src_cpuiddump); infos->apicid = ebx >> 24; if (edx & (1 << 28)) { - legacy_max_log_proc = 1 << hwloc_flsl(((ebx >> 16) & 0xff) - 1); + unsigned ebx_16_23 = (ebx >> 16) & 0xff; + if (ebx_16_23) { + legacy_max_log_proc = 1 << hwloc_flsl(ebx_16_23 - 1); + } else { + hwloc_debug("HTT bit set in CPUID 0x01.edx, but legacy_max_proc = 0 in ebx, assuming legacy_max_log_proc = 1\n"); + legacy_max_log_proc = 1; + } } else { hwloc_debug("HTT bit not set in CPUID 0x01.edx, assuming legacy_max_log_proc = 1\n"); legacy_max_log_proc = 1; @@ -1753,7 +1748,7 @@ hwloc_x86_discover(struct hwloc_backend *backend, struct hwloc_disc_status *dsta if (topology->levels[0][0]->cpuset) { /* somebody else discovered things, reconnect levels so that we can look at them */ - hwloc_topology_reconnect(topology, 0); + hwloc__reconnect(topology, 0); if (topology->nb_levels == 2 && topology->level_nbobjects[1] == data->nbprocs) { /* only PUs were discovered, as much as we would, complete the topology with everything else */ alreadypus = 1; @@ -1825,7 +1820,7 @@ hwloc_x86_check_cpuiddump_input(const char *src_cpuiddump_path, hwloc_bitmap_t s goto out_with_path; } fclose(file); - if (strcmp(line, "Architecture: x86\n")) { + if (strncmp(line, "Architecture: x86", 17)) { fprintf(stderr, "hwloc/x86: Found non-x86 dumped cpuid summary in %s: %s\n", path, line); goto out_with_path; } diff --git a/src/3rdparty/hwloc/src/topology-xml-nolibxml.c b/src/3rdparty/hwloc/src/topology-xml-nolibxml.c index 8ea5e385..e59738a9 100644 --- a/src/3rdparty/hwloc/src/topology-xml-nolibxml.c +++ b/src/3rdparty/hwloc/src/topology-xml-nolibxml.c @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2020 Inria. All rights reserved. + * Copyright © 2009-2024 Inria. All rights reserved. * Copyright © 2009-2011 Université Bordeaux * Copyright © 2009-2011 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -41,7 +41,7 @@ typedef struct hwloc__nolibxml_import_state_data_s { static char * hwloc__nolibxml_import_ignore_spaces(char *buffer) { - return buffer + strspn(buffer, " \t\n"); + return buffer + strspn(buffer, " \t\n\r"); } static int diff --git a/src/3rdparty/hwloc/src/topology-xml.c b/src/3rdparty/hwloc/src/topology-xml.c index 70006f63..67f56025 100644 --- a/src/3rdparty/hwloc/src/topology-xml.c +++ b/src/3rdparty/hwloc/src/topology-xml.c @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2023 Inria. All rights reserved. + * Copyright © 2009-2025 Inria. All rights reserved. * Copyright © 2009-2011, 2020 Université Bordeaux * Copyright © 2009-2018 Cisco Systems, Inc. All rights reserved. * See COPYING in top-level directory. @@ -415,6 +415,20 @@ hwloc__xml_import_object_attr(struct hwloc_topology *topology, } } + else if (!strcmp(name, "numanode_type")) { + switch (obj->type) { + case HWLOC_OBJ_NUMANODE: { + /* ignored for now, here for possible forward compat */ + break; + } + default: + if (hwloc__xml_verbose()) + fprintf(stderr, "%s: ignoring numanode_type attribute for non-NUMA object\n", + state->global->msgprefix); + break; + } + } + else if (data->version_major < 2) { /************************ * deprecated from 1.x @@ -872,14 +886,23 @@ hwloc__xml_import_object(hwloc_topology_t topology, /* deal with possible future type */ obj->type = HWLOC_OBJ_GROUP; obj->attr->group.kind = HWLOC_GROUP_KIND_INTEL_MODULE; - } else if (!strcasecmp(attrvalue, "MemCache")) { + } else if (!strcasecmp(attrvalue, "Cluster")) { + /* deal with possible future type */ + obj->type = HWLOC_OBJ_GROUP; + obj->attr->group.kind = HWLOC_GROUP_KIND_LINUX_CLUSTER; + } +#if 0 + /* reenable if there's ever a future type that should be ignored without being an error */ + else if (!strcasecmp(attrvalue, "MemCache")) { /* ignore possible future type */ obj->type = _HWLOC_OBJ_FUTURE; ignored = 1; if (hwloc__xml_verbose()) fprintf(stderr, "%s: %s object not-supported, will be ignored\n", state->global->msgprefix, attrvalue); - } else { + } +#endif + else { if (hwloc__xml_verbose()) fprintf(stderr, "%s: unrecognized object type string %s\n", state->global->msgprefix, attrvalue); @@ -954,22 +977,22 @@ hwloc__xml_import_object(hwloc_topology_t topology, if (hwloc__obj_type_is_normal(obj->type)) { if (!hwloc__obj_type_is_normal(parent->type)) { if (hwloc__xml_verbose()) - fprintf(stderr, "normal object %s cannot be child of non-normal parent %s\n", - hwloc_obj_type_string(obj->type), hwloc_obj_type_string(parent->type)); + fprintf(stderr, "%s: normal object %s cannot be child of non-normal parent %s\n", + state->global->msgprefix, hwloc_obj_type_string(obj->type), hwloc_obj_type_string(parent->type)); goto error_with_object; } } else if (hwloc__obj_type_is_memory(obj->type)) { if (hwloc__obj_type_is_io(parent->type) || HWLOC_OBJ_MISC == parent->type) { if (hwloc__xml_verbose()) - fprintf(stderr, "Memory object %s cannot be child of non-normal-or-memory parent %s\n", - hwloc_obj_type_string(obj->type), hwloc_obj_type_string(parent->type)); + fprintf(stderr, "%s: Memory object %s cannot be child of non-normal-or-memory parent %s\n", + state->global->msgprefix, hwloc_obj_type_string(obj->type), hwloc_obj_type_string(parent->type)); goto error_with_object; } } else if (hwloc__obj_type_is_io(obj->type)) { if (hwloc__obj_type_is_memory(parent->type) || HWLOC_OBJ_MISC == parent->type) { if (hwloc__xml_verbose()) - fprintf(stderr, "I/O object %s cannot be child of non-normal-or-I/O parent %s\n", - hwloc_obj_type_string(obj->type), hwloc_obj_type_string(parent->type)); + fprintf(stderr, "%s: I/O object %s cannot be child of non-normal-or-I/O parent %s\n", + state->global->msgprefix, hwloc_obj_type_string(obj->type), hwloc_obj_type_string(parent->type)); goto error_with_object; } } @@ -1344,7 +1367,7 @@ hwloc__xml_v2import_support(hwloc_topology_t topology, HWLOC_BUILD_ASSERT(sizeof(struct hwloc_topology_support) == 4*sizeof(void*)); HWLOC_BUILD_ASSERT(sizeof(struct hwloc_topology_discovery_support) == 6); HWLOC_BUILD_ASSERT(sizeof(struct hwloc_topology_cpubind_support) == 11); - HWLOC_BUILD_ASSERT(sizeof(struct hwloc_topology_membind_support) == 15); + HWLOC_BUILD_ASSERT(sizeof(struct hwloc_topology_membind_support) == 16); HWLOC_BUILD_ASSERT(sizeof(struct hwloc_topology_misc_support) == 1); #endif @@ -1378,6 +1401,7 @@ hwloc__xml_v2import_support(hwloc_topology_t topology, else DO(membind,firsttouch_membind); else DO(membind,bind_membind); else DO(membind,interleave_membind); + else DO(membind,weighted_interleave_membind); else DO(membind,nexttouch_membind); else DO(membind,migrate_membind); else DO(membind,get_area_memlocation); @@ -1436,6 +1460,10 @@ hwloc__xml_v2import_distances(hwloc_topology_t topology, } else if (!strcmp(attrname, "kind")) { kind = strtoul(attrvalue, NULL, 10); + /* forward compat with "HOPS" kind in v3 */ + if (kind & (1UL<<5)) + /* hops becomes latency */ + kind = (kind & ~(1UL<<5)) | HWLOC_DISTANCES_KIND_MEANS_LATENCY; } else if (!strcmp(attrname, "name")) { name = attrvalue; @@ -3087,7 +3115,7 @@ hwloc__xml_v2export_support(hwloc__xml_export_state_t parentstate, hwloc_topolog HWLOC_BUILD_ASSERT(sizeof(struct hwloc_topology_support) == 4*sizeof(void*)); HWLOC_BUILD_ASSERT(sizeof(struct hwloc_topology_discovery_support) == 6); HWLOC_BUILD_ASSERT(sizeof(struct hwloc_topology_cpubind_support) == 11); - HWLOC_BUILD_ASSERT(sizeof(struct hwloc_topology_membind_support) == 15); + HWLOC_BUILD_ASSERT(sizeof(struct hwloc_topology_membind_support) == 16); HWLOC_BUILD_ASSERT(sizeof(struct hwloc_topology_misc_support) == 1); #endif @@ -3132,6 +3160,7 @@ hwloc__xml_v2export_support(hwloc__xml_export_state_t parentstate, hwloc_topolog DO(membind,firsttouch_membind); DO(membind,bind_membind); DO(membind,interleave_membind); + DO(membind,weighted_interleave_membind); DO(membind,nexttouch_membind); DO(membind,migrate_membind); DO(membind,get_area_memlocation); diff --git a/src/3rdparty/hwloc/src/topology.c b/src/3rdparty/hwloc/src/topology.c index 9dc2b07c..2d0eb832 100644 --- a/src/3rdparty/hwloc/src/topology.c +++ b/src/3rdparty/hwloc/src/topology.c @@ -1,6 +1,6 @@ /* * Copyright © 2009 CNRS - * Copyright © 2009-2023 Inria. All rights reserved. + * Copyright © 2009-2025 Inria. All rights reserved. * Copyright © 2009-2012, 2020 Université Bordeaux * Copyright © 2009-2011 Cisco Systems, Inc. All rights reserved. * Copyright © 2022 IBM Corporation. All rights reserved. @@ -54,56 +54,6 @@ #endif -#ifdef HWLOC_HAVE_LEVELZERO -/* - * Define ZES_ENABLE_SYSMAN=1 early so that the LevelZero backend gets Sysman enabled. - * - * Only if the levelzero was enabled in this build so that we don't enable sysman - * for external levelzero users when hwloc doesn't need it. If somebody ever loads - * an external levelzero plugin in a hwloc library built without levelzero (unlikely), - * he may have to manually set ZES_ENABLE_SYSMAN=1. - * - * Use the constructor if supported and/or the Windows DllMain callback. - * Do it in the main hwloc library instead of the levelzero component because - * the latter could be loaded later as a plugin. - * - * L0 seems to be using getenv() to check this variable on Windows - * (at least in the Intel Compute-Runtime of March 2021), - * but setenv() doesn't seem to exist on Windows, hence use putenv() to set the variable. - * - * For the record, Get/SetEnvironmentVariable() is not exactly the same as getenv/putenv(): - * - getenv() doesn't see what was set with SetEnvironmentVariable() - * - GetEnvironmentVariable() doesn't see putenv() in cygwin (while it does in MSVC and MinGW). - * Hence, if L0 ever switches from getenv() to GetEnvironmentVariable(), - * it will break in cygwin, we'll have to use both putenv() and SetEnvironmentVariable(). - * Hopefully L0 will provide a way to enable Sysman without env vars before it happens. - */ -#if HWLOC_HAVE_ATTRIBUTE_CONSTRUCTOR -static void hwloc_constructor(void) __attribute__((constructor)); -static void hwloc_constructor(void) -{ - if (!getenv("ZES_ENABLE_SYSMAN")) -#ifdef HWLOC_WIN_SYS - putenv("ZES_ENABLE_SYSMAN=1"); -#else - setenv("ZES_ENABLE_SYSMAN", "1", 1); -#endif -} -#endif -#ifdef HWLOC_WIN_SYS -BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved) -{ - if (fdwReason == DLL_PROCESS_ATTACH) { - if (!getenv("ZES_ENABLE_SYSMAN")) - /* Windows does not have a setenv, so use putenv. */ - putenv((char *) "ZES_ENABLE_SYSMAN=1"); - } - return TRUE; -} -#endif -#endif /* HWLOC_HAVE_LEVELZERO */ - - unsigned hwloc_get_api_version(void) { return HWLOC_API_VERSION; @@ -179,7 +129,7 @@ static void report_insert_error(hwloc_obj_t new, hwloc_obj_t old, const char *ms report_insert_error_format_obj(oldstr, sizeof(oldstr), old); fprintf(stderr, "****************************************************************************\n"); - fprintf(stderr, "* hwloc %s received invalid information from the operating system.\n", HWLOC_VERSION); + fprintf(stderr, "* hwloc %s received invalid information.\n", HWLOC_VERSION); fprintf(stderr, "*\n"); fprintf(stderr, "* Failed with error: %s\n", msg); fprintf(stderr, "* while inserting %s\n", newstr); @@ -465,6 +415,20 @@ hwloc_debug_print_objects(int indent __hwloc_attribute_unused, hwloc_obj_t obj) #define hwloc_debug_print_objects(indent, obj) do { /* nothing */ } while (0) #endif /* !HWLOC_DEBUG */ +int hwloc_obj_set_subtype(hwloc_topology_t topology __hwloc_attribute_unused, hwloc_obj_t obj, const char *subtype) +{ + char *new = NULL; + if (subtype) { + new = strdup(subtype); + if (!new) + return -1; + } + if (obj->subtype) + free(obj->subtype); + obj->subtype = new; + return 0; +} + void hwloc__free_infos(struct hwloc_info_s *infos, unsigned count) { unsigned i; @@ -1952,6 +1916,51 @@ static void hwloc_set_group_depth(hwloc_topology_t topology); static void hwloc_connect_children(hwloc_obj_t parent); static int hwloc_connect_levels(hwloc_topology_t topology); static int hwloc_connect_special_levels(hwloc_topology_t topology); +static int hwloc_filter_levels_keep_structure(hwloc_topology_t topology); + +/* reconnect children and levels, + * and optionnally merged identical levels while keeping structure. + */ +int +hwloc__reconnect(struct hwloc_topology *topology, unsigned long flags) +{ + int merged_levels = 0; + + if (topology->modified) { + hwloc_connect_children(topology->levels[0][0]); + + if (hwloc_connect_levels(topology) < 0) + return -1; + } + + if (flags & _HWLOC_RECONNECT_FLAG_KEEPSTRUCTURE) { + merged_levels = hwloc_filter_levels_keep_structure(topology); + /* If > 0, we merged some levels, + * some child+parent special children list may have been merged, + * hence specials level might need reordering, + * So reconnect special levels only here at the end. + */ + } + + if (topology->modified || merged_levels) { + if (hwloc_connect_special_levels(topology) < 0) + return -1; + } + + topology->modified = 0; + return 0; +} + +int +hwloc_topology_reconnect(struct hwloc_topology *topology, unsigned long flags) +{ + if (flags) { + errno = EINVAL; + return -1; + } + + return hwloc__reconnect(topology, 0); +} hwloc_obj_t hwloc_topology_insert_group_object(struct hwloc_topology *topology, hwloc_obj_t obj) @@ -2044,7 +2053,10 @@ hwloc_topology_insert_group_object(struct hwloc_topology *topology, hwloc_obj_t /* properly inserted */ hwloc_obj_add_children_sets(res); - if (hwloc_topology_reconnect(topology, 0) < 0) + /* reconnect levels. + * no need to filter levels keep_structure because groups are either auto-merged + * or have the dont_merge attribute */ + if (hwloc__reconnect(topology, 0) < 0) return NULL; /* Compute group total_memory. */ @@ -2536,26 +2548,13 @@ hwloc_compare_levels_structure(hwloc_topology_t topology, unsigned i) return 0; } -/* return > 0 if any level was removed. - * performs its own reconnect internally if needed - */ +/* return > 0 if any level was removed. */ static int hwloc_filter_levels_keep_structure(hwloc_topology_t topology) { unsigned i, j; int res = 0; - if (topology->modified) { - /* WARNING: hwloc_topology_reconnect() is duplicated partially here - * and at the end of this function: - * - we need normal levels before merging. - * - and we'll need to update special levels after merging. - */ - hwloc_connect_children(topology->levels[0][0]); - if (hwloc_connect_levels(topology) < 0) - return -1; - } - /* start from the bottom since we'll remove intermediate levels */ for(i=topology->nb_levels-1; i>0; i--) { int replacechild = 0, replaceparent = 0; @@ -2577,9 +2576,15 @@ hwloc_filter_levels_keep_structure(hwloc_topology_t topology) if (type1 == HWLOC_OBJ_GROUP && hwloc_dont_merge_group_level(topology, i)) replacechild = 0; } - if (!replacechild && !replaceparent) + if (!replacechild && !replaceparent) { + /* always merge Die into Package when levels are identical */ + if (type1 == HWLOC_OBJ_PACKAGE && type2 == HWLOC_OBJ_DIE) + replacechild = 1; + } + if (!replacechild && !replaceparent) { /* no ignoring */ continue; + } /* Decide which one to actually replace */ if (replaceparent && replacechild) { /* If both may be replaced, look at obj_type_priority */ @@ -2722,20 +2727,6 @@ hwloc_filter_levels_keep_structure(hwloc_topology_t topology) } } - - if (res > 0 || topology-> modified) { - /* WARNING: hwloc_topology_reconnect() is duplicated partially here - * and at the beginning of this function. - * If we merged some levels, some child+parent special children lisst - * may have been merged, hence specials level might need reordering, - * So reconnect special levels only here at the end - * (it's not needed at the beginning of this function). - */ - if (hwloc_connect_special_levels(topology) < 0) - return -1; - topology->modified = 0; - } - return 0; } @@ -3264,33 +3255,6 @@ hwloc_connect_levels(hwloc_topology_t topology) return 0; } -int -hwloc_topology_reconnect(struct hwloc_topology *topology, unsigned long flags) -{ - /* WARNING: when updating this function, the replicated code must - * also be updated inside hwloc_filter_levels_keep_structure() - */ - - if (flags) { - errno = EINVAL; - return -1; - } - if (!topology->modified) - return 0; - - hwloc_connect_children(topology->levels[0][0]); - - if (hwloc_connect_levels(topology) < 0) - return -1; - - if (hwloc_connect_special_levels(topology) < 0) - return -1; - - topology->modified = 0; - - return 0; -} - /* for regression testing, make sure the order of io devices * doesn't change with the dentry order in the filesystem * @@ -3547,32 +3511,13 @@ hwloc_discover(struct hwloc_topology *topology, hwloc_debug_print_objects(0, topology->levels[0][0]); } - /* see if we should ignore the root now that we know how many children it has */ - if (!hwloc_filter_check_keep_object(topology, topology->levels[0][0]) - && topology->levels[0][0]->first_child && !topology->levels[0][0]->first_child->next_sibling) { - hwloc_obj_t oldroot = topology->levels[0][0]; - hwloc_obj_t newroot = oldroot->first_child; - /* switch to the new root */ - newroot->parent = NULL; - topology->levels[0][0] = newroot; - /* move oldroot memory/io/misc children before newroot children */ - if (oldroot->memory_first_child) - prepend_siblings_list(&newroot->memory_first_child, oldroot->memory_first_child, newroot); - if (oldroot->io_first_child) - prepend_siblings_list(&newroot->io_first_child, oldroot->io_first_child, newroot); - if (oldroot->misc_first_child) - prepend_siblings_list(&newroot->misc_first_child, oldroot->misc_first_child, newroot); - /* destroy oldroot and use the new one */ - hwloc_free_unlinked_object(oldroot); - } - /* * All object cpusets and nodesets are properly set now. */ /* Now connect handy pointers to make remaining discovery easier. */ hwloc_debug("%s", "\nOk, finished tweaking, now connect\n"); - if (hwloc_topology_reconnect(topology, 0) < 0) + if (hwloc__reconnect(topology, 0) < 0) return -1; hwloc_debug_print_objects(0, topology->levels[0][0]); @@ -3628,12 +3573,12 @@ hwloc_discover(struct hwloc_topology *topology, } hwloc_debug_print_objects(0, topology->levels[0][0]); + /* reconnect all (new groups might have appears, IO added, etc), + * and (now that everything was added) remove identical levels while keeping structure + */ hwloc_debug("%s", "\nRemoving levels with HWLOC_TYPE_FILTER_KEEP_STRUCTURE\n"); - if (hwloc_filter_levels_keep_structure(topology) < 0) + if (hwloc__reconnect(topology, _HWLOC_RECONNECT_FLAG_KEEPSTRUCTURE) < 0) return -1; - /* takes care of reconnecting children/levels internally, - * because it needs normal levels. - * and it's often needed below because of Groups inserted for I/Os anyway */ hwloc_debug_print_objects(0, topology->levels[0][0]); /* accumulate children memory in total_memory fields (only once parent is set) */ @@ -4480,7 +4425,7 @@ hwloc_topology_restrict(struct hwloc_topology *topology, hwloc_const_bitmap_t se hwloc_bitmap_free(droppedcpuset); hwloc_bitmap_free(droppednodeset); - if (hwloc_filter_levels_keep_structure(topology) < 0) /* takes care of reconnecting internally */ + if (hwloc__reconnect(topology, _HWLOC_RECONNECT_FLAG_KEEPSTRUCTURE) < 0) goto out; /* some objects may have disappeared and sets were modified, @@ -5102,6 +5047,8 @@ hwloc_topology_check(struct hwloc_topology *topology) unsigned i; int j, depth; + assert(!topology->modified); + /* make sure we can use ranges to check types */ /* hwloc__obj_type_is_{,d,i}cache() want cache types to be ordered like this */ diff --git a/src/3rdparty/libethash/CMakeLists.txt b/src/3rdparty/libethash/CMakeLists.txt index 491432a6..51993e0d 100644 --- a/src/3rdparty/libethash/CMakeLists.txt +++ b/src/3rdparty/libethash/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.10) project (ethash C) set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -Os") diff --git a/src/3rdparty/llhttp/api.c b/src/3rdparty/llhttp/api.c index 8a4bdd26..02452541 100644 --- a/src/3rdparty/llhttp/api.c +++ b/src/3rdparty/llhttp/api.c @@ -4,7 +4,7 @@ #include "llhttp.h" -#define CALLBACK_MAYBE(PARSER, NAME, ...) \ +#define CALLBACK_MAYBE(PARSER, NAME) \ do { \ const llhttp_settings_t* settings; \ settings = (const llhttp_settings_t*) (PARSER)->settings; \ @@ -12,7 +12,22 @@ err = 0; \ break; \ } \ - err = settings->NAME(__VA_ARGS__); \ + err = settings->NAME((PARSER)); \ + } while (0) + +#define SPAN_CALLBACK_MAYBE(PARSER, NAME, START, LEN) \ + do { \ + const llhttp_settings_t* settings; \ + settings = (const llhttp_settings_t*) (PARSER)->settings; \ + if (settings == NULL || settings->NAME == NULL) { \ + err = 0; \ + break; \ + } \ + err = settings->NAME((PARSER), (START), (LEN)); \ + if (err == -1) { \ + err = HPE_USER; \ + llhttp_set_error_reason((PARSER), "Span callback error in " #NAME); \ + } \ } while (0) void llhttp_init(llhttp_t* parser, llhttp_type_t type, @@ -31,21 +46,25 @@ extern int wasm_on_url(llhttp_t* p, const char* at, size_t length); extern int wasm_on_status(llhttp_t* p, const char* at, size_t length); extern int wasm_on_header_field(llhttp_t* p, const char* at, size_t length); extern int wasm_on_header_value(llhttp_t* p, const char* at, size_t length); -extern int wasm_on_headers_complete(llhttp_t * p); +extern int wasm_on_headers_complete(llhttp_t * p, int status_code, + uint8_t upgrade, int should_keep_alive); extern int wasm_on_body(llhttp_t* p, const char* at, size_t length); extern int wasm_on_message_complete(llhttp_t * p); +static int wasm_on_headers_complete_wrap(llhttp_t* p) { + return wasm_on_headers_complete(p, p->status_code, p->upgrade, + llhttp_should_keep_alive(p)); +} + const llhttp_settings_t wasm_settings = { - wasm_on_message_begin, - wasm_on_url, - wasm_on_status, - wasm_on_header_field, - wasm_on_header_value, - wasm_on_headers_complete, - wasm_on_body, - wasm_on_message_complete, - NULL, - NULL, + .on_message_begin = wasm_on_message_begin, + .on_url = wasm_on_url, + .on_status = wasm_on_status, + .on_header_field = wasm_on_header_field, + .on_header_value = wasm_on_header_value, + .on_headers_complete = wasm_on_headers_complete_wrap, + .on_body = wasm_on_body, + .on_message_complete = wasm_on_message_complete, }; @@ -59,6 +78,8 @@ void llhttp_free(llhttp_t* parser) { free(parser); } +#endif // defined(__wasm__) + /* Some getters required to get stuff from the parser */ uint8_t llhttp_get_type(llhttp_t* parser) { @@ -85,14 +106,12 @@ uint8_t llhttp_get_upgrade(llhttp_t* parser) { return parser->upgrade; } -#endif // defined(__wasm__) - void llhttp_reset(llhttp_t* parser) { llhttp_type_t type = parser->type; const llhttp_settings_t* settings = parser->settings; void* data = parser->data; - uint8_t lenient_flags = parser->lenient_flags; + uint16_t lenient_flags = parser->lenient_flags; llhttp__internal_init(parser); @@ -123,7 +142,7 @@ llhttp_errno_t llhttp_finish(llhttp_t* parser) { switch (parser->finish) { case HTTP_FINISH_SAFE_WITH_CB: - CALLBACK_MAYBE(parser, on_message_complete, parser); + CALLBACK_MAYBE(parser, on_message_complete); if (err != HPE_OK) return err; /* FALLTHROUGH */ @@ -199,12 +218,21 @@ const char* llhttp_errno_name(llhttp_errno_t err) { const char* llhttp_method_name(llhttp_method_t method) { #define HTTP_METHOD_GEN(NUM, NAME, STRING) case HTTP_##NAME: return #STRING; switch (method) { - HTTP_METHOD_MAP(HTTP_METHOD_GEN) + HTTP_ALL_METHOD_MAP(HTTP_METHOD_GEN) default: abort(); } #undef HTTP_METHOD_GEN } +const char* llhttp_status_name(llhttp_status_t status) { +#define HTTP_STATUS_GEN(NUM, NAME, STRING) case HTTP_STATUS_##NAME: return #STRING; + switch (status) { + HTTP_STATUS_MAP(HTTP_STATUS_GEN) + default: abort(); + } +#undef HTTP_STATUS_GEN +} + void llhttp_set_lenient_headers(llhttp_t* parser, int enabled) { if (enabled) { @@ -232,103 +260,236 @@ void llhttp_set_lenient_keep_alive(llhttp_t* parser, int enabled) { } } +void llhttp_set_lenient_transfer_encoding(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_TRANSFER_ENCODING; + } else { + parser->lenient_flags &= ~LENIENT_TRANSFER_ENCODING; + } +} + +void llhttp_set_lenient_version(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_VERSION; + } else { + parser->lenient_flags &= ~LENIENT_VERSION; + } +} + +void llhttp_set_lenient_data_after_close(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_DATA_AFTER_CLOSE; + } else { + parser->lenient_flags &= ~LENIENT_DATA_AFTER_CLOSE; + } +} + +void llhttp_set_lenient_optional_lf_after_cr(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_OPTIONAL_LF_AFTER_CR; + } else { + parser->lenient_flags &= ~LENIENT_OPTIONAL_LF_AFTER_CR; + } +} + +void llhttp_set_lenient_optional_crlf_after_chunk(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_OPTIONAL_CRLF_AFTER_CHUNK; + } else { + parser->lenient_flags &= ~LENIENT_OPTIONAL_CRLF_AFTER_CHUNK; + } +} + +void llhttp_set_lenient_optional_cr_before_lf(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_OPTIONAL_CR_BEFORE_LF; + } else { + parser->lenient_flags &= ~LENIENT_OPTIONAL_CR_BEFORE_LF; + } +} + +void llhttp_set_lenient_spaces_after_chunk_size(llhttp_t* parser, int enabled) { + if (enabled) { + parser->lenient_flags |= LENIENT_SPACES_AFTER_CHUNK_SIZE; + } else { + parser->lenient_flags &= ~LENIENT_SPACES_AFTER_CHUNK_SIZE; + } +} + /* Callbacks */ int llhttp__on_message_begin(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_message_begin, s); + CALLBACK_MAYBE(s, on_message_begin); + return err; +} + + +int llhttp__on_protocol(llhttp_t* s, const char* p, const char* endp) { + int err; + SPAN_CALLBACK_MAYBE(s, on_protocol, p, endp - p); + return err; +} + + +int llhttp__on_protocol_complete(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_protocol_complete); return err; } int llhttp__on_url(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_url, s, p, endp - p); + SPAN_CALLBACK_MAYBE(s, on_url, p, endp - p); return err; } int llhttp__on_url_complete(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_url_complete, s); + CALLBACK_MAYBE(s, on_url_complete); return err; } int llhttp__on_status(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_status, s, p, endp - p); + SPAN_CALLBACK_MAYBE(s, on_status, p, endp - p); return err; } int llhttp__on_status_complete(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_status_complete, s); + CALLBACK_MAYBE(s, on_status_complete); + return err; +} + + +int llhttp__on_method(llhttp_t* s, const char* p, const char* endp) { + int err; + SPAN_CALLBACK_MAYBE(s, on_method, p, endp - p); + return err; +} + + +int llhttp__on_method_complete(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_method_complete); + return err; +} + + +int llhttp__on_version(llhttp_t* s, const char* p, const char* endp) { + int err; + SPAN_CALLBACK_MAYBE(s, on_version, p, endp - p); + return err; +} + + +int llhttp__on_version_complete(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_version_complete); return err; } int llhttp__on_header_field(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_header_field, s, p, endp - p); + SPAN_CALLBACK_MAYBE(s, on_header_field, p, endp - p); return err; } int llhttp__on_header_field_complete(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_header_field_complete, s); + CALLBACK_MAYBE(s, on_header_field_complete); return err; } int llhttp__on_header_value(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_header_value, s, p, endp - p); + SPAN_CALLBACK_MAYBE(s, on_header_value, p, endp - p); return err; } int llhttp__on_header_value_complete(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_header_value_complete, s); + CALLBACK_MAYBE(s, on_header_value_complete); return err; } int llhttp__on_headers_complete(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_headers_complete, s); + CALLBACK_MAYBE(s, on_headers_complete); return err; } int llhttp__on_message_complete(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_message_complete, s); + CALLBACK_MAYBE(s, on_message_complete); return err; } int llhttp__on_body(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_body, s, p, endp - p); + SPAN_CALLBACK_MAYBE(s, on_body, p, endp - p); return err; } int llhttp__on_chunk_header(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_chunk_header, s); + CALLBACK_MAYBE(s, on_chunk_header); + return err; +} + + +int llhttp__on_chunk_extension_name(llhttp_t* s, const char* p, const char* endp) { + int err; + SPAN_CALLBACK_MAYBE(s, on_chunk_extension_name, p, endp - p); + return err; +} + + +int llhttp__on_chunk_extension_name_complete(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_chunk_extension_name_complete); + return err; +} + + +int llhttp__on_chunk_extension_value(llhttp_t* s, const char* p, const char* endp) { + int err; + SPAN_CALLBACK_MAYBE(s, on_chunk_extension_value, p, endp - p); + return err; +} + + +int llhttp__on_chunk_extension_value_complete(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_chunk_extension_value_complete); return err; } int llhttp__on_chunk_complete(llhttp_t* s, const char* p, const char* endp) { int err; - CALLBACK_MAYBE(s, on_chunk_complete, s); + CALLBACK_MAYBE(s, on_chunk_complete); + return err; +} + + +int llhttp__on_reset(llhttp_t* s, const char* p, const char* endp) { + int err; + CALLBACK_MAYBE(s, on_reset); return err; } diff --git a/src/3rdparty/llhttp/api.h b/src/3rdparty/llhttp/api.h deleted file mode 100644 index 7fdbfc10..00000000 --- a/src/3rdparty/llhttp/api.h +++ /dev/null @@ -1,253 +0,0 @@ -#ifndef INCLUDE_LLHTTP_API_H_ -#define INCLUDE_LLHTTP_API_H_ -#ifdef __cplusplus -extern "C" { -#endif -#include - -#if defined(__wasm__) -#define LLHTTP_EXPORT __attribute__((visibility("default"))) -#else -#define LLHTTP_EXPORT -#endif - -typedef llhttp__internal_t llhttp_t; -typedef struct llhttp_settings_s llhttp_settings_t; - -typedef int (*llhttp_data_cb)(llhttp_t*, const char *at, size_t length); -typedef int (*llhttp_cb)(llhttp_t*); - -struct llhttp_settings_s { - /* Possible return values 0, -1, `HPE_PAUSED` */ - llhttp_cb on_message_begin; - - llhttp_data_cb on_url; - llhttp_data_cb on_status; - llhttp_data_cb on_header_field; - llhttp_data_cb on_header_value; - - /* Possible return values: - * 0 - Proceed normally - * 1 - Assume that request/response has no body, and proceed to parsing the - * next message - * 2 - Assume absence of body (as above) and make `llhttp_execute()` return - * `HPE_PAUSED_UPGRADE` - * -1 - Error - * `HPE_PAUSED` - */ - llhttp_cb on_headers_complete; - - llhttp_data_cb on_body; - - /* Possible return values 0, -1, `HPE_PAUSED` */ - llhttp_cb on_message_complete; - - /* When on_chunk_header is called, the current chunk length is stored - * in parser->content_length. - * Possible return values 0, -1, `HPE_PAUSED` - */ - llhttp_cb on_chunk_header; - llhttp_cb on_chunk_complete; - - llhttp_cb on_url_complete; - llhttp_cb on_status_complete; - llhttp_cb on_header_field_complete; - llhttp_cb on_header_value_complete; -}; - -/* Initialize the parser with specific type and user settings. - * - * NOTE: lifetime of `settings` has to be at least the same as the lifetime of - * the `parser` here. In practice, `settings` has to be either a static - * variable or be allocated with `malloc`, `new`, etc. - */ -LLHTTP_EXPORT -void llhttp_init(llhttp_t* parser, llhttp_type_t type, - const llhttp_settings_t* settings); - -#if defined(__wasm__) - -LLHTTP_EXPORT -llhttp_t* llhttp_alloc(llhttp_type_t type); - -LLHTTP_EXPORT -void llhttp_free(llhttp_t* parser); - -LLHTTP_EXPORT -uint8_t llhttp_get_type(llhttp_t* parser); - -LLHTTP_EXPORT -uint8_t llhttp_get_http_major(llhttp_t* parser); - -LLHTTP_EXPORT -uint8_t llhttp_get_http_minor(llhttp_t* parser); - -LLHTTP_EXPORT -uint8_t llhttp_get_method(llhttp_t* parser); - -LLHTTP_EXPORT -int llhttp_get_status_code(llhttp_t* parser); - -LLHTTP_EXPORT -uint8_t llhttp_get_upgrade(llhttp_t* parser); - -#endif // defined(__wasm__) - -/* Reset an already initialized parser back to the start state, preserving the - * existing parser type, callback settings, user data, and lenient flags. - */ -LLHTTP_EXPORT -void llhttp_reset(llhttp_t* parser); - -/* Initialize the settings object */ -LLHTTP_EXPORT -void llhttp_settings_init(llhttp_settings_t* settings); - -/* Parse full or partial request/response, invoking user callbacks along the - * way. - * - * If any of `llhttp_data_cb` returns errno not equal to `HPE_OK` - the parsing - * interrupts, and such errno is returned from `llhttp_execute()`. If - * `HPE_PAUSED` was used as a errno, the execution can be resumed with - * `llhttp_resume()` call. - * - * In a special case of CONNECT/Upgrade request/response `HPE_PAUSED_UPGRADE` - * is returned after fully parsing the request/response. If the user wishes to - * continue parsing, they need to invoke `llhttp_resume_after_upgrade()`. - * - * NOTE: if this function ever returns a non-pause type error, it will continue - * to return the same error upon each successive call up until `llhttp_init()` - * is called. - */ -LLHTTP_EXPORT -llhttp_errno_t llhttp_execute(llhttp_t* parser, const char* data, size_t len); - -/* This method should be called when the other side has no further bytes to - * send (e.g. shutdown of readable side of the TCP connection.) - * - * Requests without `Content-Length` and other messages might require treating - * all incoming bytes as the part of the body, up to the last byte of the - * connection. This method will invoke `on_message_complete()` callback if the - * request was terminated safely. Otherwise a error code would be returned. - */ -LLHTTP_EXPORT -llhttp_errno_t llhttp_finish(llhttp_t* parser); - -/* Returns `1` if the incoming message is parsed until the last byte, and has - * to be completed by calling `llhttp_finish()` on EOF - */ -LLHTTP_EXPORT -int llhttp_message_needs_eof(const llhttp_t* parser); - -/* Returns `1` if there might be any other messages following the last that was - * successfully parsed. - */ -LLHTTP_EXPORT -int llhttp_should_keep_alive(const llhttp_t* parser); - -/* Make further calls of `llhttp_execute()` return `HPE_PAUSED` and set - * appropriate error reason. - * - * Important: do not call this from user callbacks! User callbacks must return - * `HPE_PAUSED` if pausing is required. - */ -LLHTTP_EXPORT -void llhttp_pause(llhttp_t* parser); - -/* Might be called to resume the execution after the pause in user's callback. - * See `llhttp_execute()` above for details. - * - * Call this only if `llhttp_execute()` returns `HPE_PAUSED`. - */ -LLHTTP_EXPORT -void llhttp_resume(llhttp_t* parser); - -/* Might be called to resume the execution after the pause in user's callback. - * See `llhttp_execute()` above for details. - * - * Call this only if `llhttp_execute()` returns `HPE_PAUSED_UPGRADE` - */ -LLHTTP_EXPORT -void llhttp_resume_after_upgrade(llhttp_t* parser); - -/* Returns the latest return error */ -LLHTTP_EXPORT -llhttp_errno_t llhttp_get_errno(const llhttp_t* parser); - -/* Returns the verbal explanation of the latest returned error. - * - * Note: User callback should set error reason when returning the error. See - * `llhttp_set_error_reason()` for details. - */ -LLHTTP_EXPORT -const char* llhttp_get_error_reason(const llhttp_t* parser); - -/* Assign verbal description to the returned error. Must be called in user - * callbacks right before returning the errno. - * - * Note: `HPE_USER` error code might be useful in user callbacks. - */ -LLHTTP_EXPORT -void llhttp_set_error_reason(llhttp_t* parser, const char* reason); - -/* Returns the pointer to the last parsed byte before the returned error. The - * pointer is relative to the `data` argument of `llhttp_execute()`. - * - * Note: this method might be useful for counting the number of parsed bytes. - */ -LLHTTP_EXPORT -const char* llhttp_get_error_pos(const llhttp_t* parser); - -/* Returns textual name of error code */ -LLHTTP_EXPORT -const char* llhttp_errno_name(llhttp_errno_t err); - -/* Returns textual name of HTTP method */ -LLHTTP_EXPORT -const char* llhttp_method_name(llhttp_method_t method); - - -/* Enables/disables lenient header value parsing (disabled by default). - * - * Lenient parsing disables header value token checks, extending llhttp's - * protocol support to highly non-compliant clients/server. No - * `HPE_INVALID_HEADER_TOKEN` will be raised for incorrect header values when - * lenient parsing is "on". - * - * **(USE AT YOUR OWN RISK)** - */ -LLHTTP_EXPORT -void llhttp_set_lenient_headers(llhttp_t* parser, int enabled); - - -/* Enables/disables lenient handling of conflicting `Transfer-Encoding` and - * `Content-Length` headers (disabled by default). - * - * Normally `llhttp` would error when `Transfer-Encoding` is present in - * conjunction with `Content-Length`. This error is important to prevent HTTP - * request smuggling, but may be less desirable for small number of cases - * involving legacy servers. - * - * **(USE AT YOUR OWN RISK)** - */ -LLHTTP_EXPORT -void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled); - - -/* Enables/disables lenient handling of `Connection: close` and HTTP/1.0 - * requests responses. - * - * Normally `llhttp` would error on (in strict mode) or discard (in loose mode) - * the HTTP request/response after the request/response with `Connection: close` - * and `Content-Length`. This is important to prevent cache poisoning attacks, - * but might interact badly with outdated and insecure clients. With this flag - * the extra request/response will be parsed normally. - * - * **(USE AT YOUR OWN RISK)** - */ -void llhttp_set_lenient_keep_alive(llhttp_t* parser, int enabled); - -#ifdef __cplusplus -} /* extern "C" */ -#endif -#endif /* INCLUDE_LLHTTP_API_H_ */ diff --git a/src/3rdparty/llhttp/http.c b/src/3rdparty/llhttp/http.c index e6279f3b..1ab91a55 100644 --- a/src/3rdparty/llhttp/http.c +++ b/src/3rdparty/llhttp/http.c @@ -39,20 +39,41 @@ int llhttp__after_headers_complete(llhttp_t* parser, const char* p, int hasBody; hasBody = parser->flags & F_CHUNKED || parser->content_length > 0; - if (parser->upgrade && (parser->method == HTTP_CONNECT || - (parser->flags & F_SKIPBODY) || !hasBody)) { + if ( + (parser->upgrade && (parser->method == HTTP_CONNECT || + (parser->flags & F_SKIPBODY) || !hasBody)) || + /* See RFC 2616 section 4.4 - 1xx e.g. Continue */ + (parser->type == HTTP_RESPONSE && parser->status_code == 101) + ) { /* Exit, the rest of the message is in a different protocol. */ return 1; } - if (parser->flags & F_SKIPBODY) { + if (parser->type == HTTP_RESPONSE && parser->status_code == 100) { + /* No body, restart as the message is complete */ + return 0; + } + + /* See RFC 2616 section 4.4 */ + if ( + parser->flags & F_SKIPBODY || /* response to a HEAD request */ + ( + parser->type == HTTP_RESPONSE && ( + parser->status_code == 102 || /* Processing */ + parser->status_code == 103 || /* Early Hints */ + parser->status_code == 204 || /* No Content */ + parser->status_code == 304 /* Not Modified */ + ) + ) + ) { return 0; } else if (parser->flags & F_CHUNKED) { /* chunked encoding - ignore Content-Length header, prepare for a chunk */ return 2; } else if (parser->flags & F_TRANSFER_ENCODING) { if (parser->type == HTTP_REQUEST && - (parser->lenient_flags & LENIENT_CHUNKED_LENGTH) == 0) { + (parser->lenient_flags & LENIENT_CHUNKED_LENGTH) == 0 && + (parser->lenient_flags & LENIENT_TRANSFER_ENCODING) == 0) { /* RFC 7230 3.3.3 */ /* If a Transfer-Encoding header field diff --git a/src/3rdparty/llhttp/llhttp.c b/src/3rdparty/llhttp/llhttp.c index 022d099d..aa4c4682 100644 --- a/src/3rdparty/llhttp/llhttp.c +++ b/src/3rdparty/llhttp/llhttp.c @@ -1,5 +1,3 @@ -#if LLHTTP_STRICT_MODE - #include #include #include @@ -12,10 +10,20 @@ #endif /* _MSC_VER */ #endif /* __SSE4_2__ */ +#ifdef __ARM_NEON__ + #include +#endif /* __ARM_NEON__ */ + +#ifdef __wasm__ + #include +#endif /* __wasm__ */ + #ifdef _MSC_VER #define ALIGN(n) _declspec(align(n)) + #define UNREACHABLE __assume(0) #else /* !_MSC_VER */ #define ALIGN(n) __attribute__((aligned(n))) + #define UNREACHABLE __builtin_unreachable() #endif /* _MSC_VER */ #include "llhttp.h" @@ -24,148 +32,148 @@ typedef int (*llhttp__internal__span_cb)( llhttp__internal_t*, const char*, const char*); static const unsigned char llparse_blob0[] = { - 0xd, 0xa -}; -static const unsigned char llparse_blob1[] = { 'o', 'n' }; -static const unsigned char llparse_blob2[] = { +static const unsigned char llparse_blob1[] = { 'e', 'c', 't', 'i', 'o', 'n' }; -static const unsigned char llparse_blob3[] = { +static const unsigned char llparse_blob2[] = { 'l', 'o', 's', 'e' }; -static const unsigned char llparse_blob4[] = { +static const unsigned char llparse_blob3[] = { 'e', 'e', 'p', '-', 'a', 'l', 'i', 'v', 'e' }; -static const unsigned char llparse_blob5[] = { +static const unsigned char llparse_blob4[] = { 'p', 'g', 'r', 'a', 'd', 'e' }; -static const unsigned char llparse_blob6[] = { +static const unsigned char llparse_blob5[] = { 'c', 'h', 'u', 'n', 'k', 'e', 'd' }; #ifdef __SSE4_2__ -static const unsigned char ALIGN(16) llparse_blob7[] = { +static const unsigned char ALIGN(16) llparse_blob6[] = { 0x9, 0x9, ' ', '~', 0x80, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; #endif /* __SSE4_2__ */ #ifdef __SSE4_2__ -static const unsigned char ALIGN(16) llparse_blob8[] = { +static const unsigned char ALIGN(16) llparse_blob7[] = { '!', '!', '#', '\'', '*', '+', '-', '.', '0', '9', 'A', 'Z', '^', 'z', '|', '|' }; #endif /* __SSE4_2__ */ #ifdef __SSE4_2__ -static const unsigned char ALIGN(16) llparse_blob9[] = { +static const unsigned char ALIGN(16) llparse_blob8[] = { '~', '~', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; #endif /* __SSE4_2__ */ -static const unsigned char llparse_blob10[] = { +static const unsigned char llparse_blob9[] = { 'e', 'n', 't', '-', 'l', 'e', 'n', 'g', 't', 'h' }; -static const unsigned char llparse_blob11[] = { +static const unsigned char llparse_blob10[] = { 'r', 'o', 'x', 'y', '-', 'c', 'o', 'n', 'n', 'e', 'c', 't', 'i', 'o', 'n' }; -static const unsigned char llparse_blob12[] = { +static const unsigned char llparse_blob11[] = { 'r', 'a', 'n', 's', 'f', 'e', 'r', '-', 'e', 'n', 'c', 'o', 'd', 'i', 'n', 'g' }; -static const unsigned char llparse_blob13[] = { +static const unsigned char llparse_blob12[] = { 'p', 'g', 'r', 'a', 'd', 'e' }; -static const unsigned char llparse_blob14[] = { - 'T', 'T', 'P', '/' +static const unsigned char llparse_blob13[] = { + 'T', 'T', 'P' }; -static const unsigned char llparse_blob15[] = { +static const unsigned char llparse_blob14[] = { 0xd, 0xa, 0xd, 0xa, 'S', 'M', 0xd, 0xa, 0xd, 0xa }; +static const unsigned char llparse_blob15[] = { + 'C', 'E' +}; static const unsigned char llparse_blob16[] = { - 'C', 'E', '/' + 'T', 'S', 'P' }; static const unsigned char llparse_blob17[] = { - 'T', 'S', 'P', '/' -}; -static const unsigned char llparse_blob18[] = { 'N', 'O', 'U', 'N', 'C', 'E' }; -static const unsigned char llparse_blob19[] = { +static const unsigned char llparse_blob18[] = { 'I', 'N', 'D' }; -static const unsigned char llparse_blob20[] = { +static const unsigned char llparse_blob19[] = { 'E', 'C', 'K', 'O', 'U', 'T' }; -static const unsigned char llparse_blob21[] = { +static const unsigned char llparse_blob20[] = { 'N', 'E', 'C', 'T' }; -static const unsigned char llparse_blob22[] = { +static const unsigned char llparse_blob21[] = { 'E', 'T', 'E' }; -static const unsigned char llparse_blob23[] = { +static const unsigned char llparse_blob22[] = { 'C', 'R', 'I', 'B', 'E' }; -static const unsigned char llparse_blob24[] = { +static const unsigned char llparse_blob23[] = { 'L', 'U', 'S', 'H' }; -static const unsigned char llparse_blob25[] = { +static const unsigned char llparse_blob24[] = { 'E', 'T' }; -static const unsigned char llparse_blob26[] = { +static const unsigned char llparse_blob25[] = { 'P', 'A', 'R', 'A', 'M', 'E', 'T', 'E', 'R' }; -static const unsigned char llparse_blob27[] = { +static const unsigned char llparse_blob26[] = { 'E', 'A', 'D' }; -static const unsigned char llparse_blob28[] = { +static const unsigned char llparse_blob27[] = { 'N', 'K' }; -static const unsigned char llparse_blob29[] = { +static const unsigned char llparse_blob28[] = { 'C', 'K' }; -static const unsigned char llparse_blob30[] = { +static const unsigned char llparse_blob29[] = { 'S', 'E', 'A', 'R', 'C', 'H' }; -static const unsigned char llparse_blob31[] = { +static const unsigned char llparse_blob30[] = { 'R', 'G', 'E' }; -static const unsigned char llparse_blob32[] = { +static const unsigned char llparse_blob31[] = { 'C', 'T', 'I', 'V', 'I', 'T', 'Y' }; -static const unsigned char llparse_blob33[] = { +static const unsigned char llparse_blob32[] = { 'L', 'E', 'N', 'D', 'A', 'R' }; -static const unsigned char llparse_blob34[] = { +static const unsigned char llparse_blob33[] = { 'V', 'E' }; -static const unsigned char llparse_blob35[] = { +static const unsigned char llparse_blob34[] = { 'O', 'T', 'I', 'F', 'Y' }; -static const unsigned char llparse_blob36[] = { +static const unsigned char llparse_blob35[] = { 'P', 'T', 'I', 'O', 'N', 'S' }; -static const unsigned char llparse_blob37[] = { +static const unsigned char llparse_blob36[] = { 'C', 'H' }; -static const unsigned char llparse_blob38[] = { +static const unsigned char llparse_blob37[] = { 'S', 'E' }; -static const unsigned char llparse_blob39[] = { +static const unsigned char llparse_blob38[] = { 'A', 'Y' }; -static const unsigned char llparse_blob40[] = { +static const unsigned char llparse_blob39[] = { 'S', 'T' }; -static const unsigned char llparse_blob41[] = { +static const unsigned char llparse_blob40[] = { 'I', 'N', 'D' }; -static const unsigned char llparse_blob42[] = { +static const unsigned char llparse_blob41[] = { 'A', 'T', 'C', 'H' }; -static const unsigned char llparse_blob43[] = { +static const unsigned char llparse_blob42[] = { 'G', 'E' }; +static const unsigned char llparse_blob43[] = { + 'U', 'E', 'R', 'Y' +}; static const unsigned char llparse_blob44[] = { 'I', 'N', 'D' }; @@ -209,12 +217,18 @@ static const unsigned char llparse_blob57[] = { 'U', 'B', 'S', 'C', 'R', 'I', 'B', 'E' }; static const unsigned char llparse_blob58[] = { - 'H', 'T', 'T', 'P', '/' + 'T', 'T', 'P' }; static const unsigned char llparse_blob59[] = { - 'A', 'D' + 'C', 'E' }; static const unsigned char llparse_blob60[] = { + 'T', 'S', 'P' +}; +static const unsigned char llparse_blob61[] = { + 'A', 'D' +}; +static const unsigned char llparse_blob62[] = { 'T', 'P', '/' }; @@ -231,38 +245,6 @@ struct llparse_match_s { }; typedef struct llparse_match_s llparse_match_t; -static llparse_match_t llparse__match_sequence_id( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp, - const unsigned char* seq, uint32_t seq_len) { - uint32_t index; - llparse_match_t res; - - index = s->_index; - for (; p != endp; p++) { - unsigned char current; - - current = *p; - if (current == seq[index]) { - if (++index == seq_len) { - res.status = kMatchComplete; - goto reset; - } - } else { - res.status = kMatchMismatch; - goto reset; - } - } - s->_index = index; - res.status = kMatchPause; - res.current = p; - return res; -reset: - s->_index = 0; - res.current = p; - return res; -} - static llparse_match_t llparse__match_sequence_to_lower( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp, @@ -327,6 +309,38 @@ reset: return res; } +static llparse_match_t llparse__match_sequence_id( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp, + const unsigned char* seq, uint32_t seq_len) { + uint32_t index; + llparse_match_t res; + + index = s->_index; + for (; p != endp; p++) { + unsigned char current; + + current = *p; + if (current == seq[index]) { + if (++index == seq_len) { + res.status = kMatchComplete; + goto reset; + } + } else { + res.status = kMatchMismatch; + goto reset; + } + } + s->_index = index; + res.status = kMatchPause; + res.current = p; + return res; +reset: + s->_index = 0; + res.current = p; + return res; +} + enum llparse_state_e { s_error, s_n_llhttp__internal__n_closed, @@ -334,29 +348,54 @@ enum llparse_state_e { s_n_llhttp__internal__n_pause_1, s_n_llhttp__internal__n_invoke_is_equal_upgrade, s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2, + s_n_llhttp__internal__n_chunk_data_almost_done_1, s_n_llhttp__internal__n_chunk_data_almost_done, s_n_llhttp__internal__n_consume_content_length, s_n_llhttp__internal__n_span_start_llhttp__on_body, s_n_llhttp__internal__n_invoke_is_equal_content_length, s_n_llhttp__internal__n_chunk_size_almost_done, - s_n_llhttp__internal__n_chunk_parameters, + s_n_llhttp__internal__n_invoke_test_lenient_flags_9, + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete, + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_1, + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_2, + s_n_llhttp__internal__n_invoke_test_lenient_flags_10, + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete, + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1, + s_n_llhttp__internal__n_chunk_extension_quoted_value_done, + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2, + s_n_llhttp__internal__n_error_30, + s_n_llhttp__internal__n_chunk_extension_quoted_value_quoted_pair, + s_n_llhttp__internal__n_error_31, + s_n_llhttp__internal__n_chunk_extension_quoted_value, + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_3, + s_n_llhttp__internal__n_error_33, + s_n_llhttp__internal__n_chunk_extension_value, + s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_value, + s_n_llhttp__internal__n_error_34, + s_n_llhttp__internal__n_chunk_extension_name, + s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_name, + s_n_llhttp__internal__n_chunk_extensions, s_n_llhttp__internal__n_chunk_size_otherwise, s_n_llhttp__internal__n_chunk_size, s_n_llhttp__internal__n_chunk_size_digit, - s_n_llhttp__internal__n_invoke_update_content_length, + s_n_llhttp__internal__n_invoke_update_content_length_1, s_n_llhttp__internal__n_consume_content_length_1, s_n_llhttp__internal__n_span_start_llhttp__on_body_1, s_n_llhttp__internal__n_eof, s_n_llhttp__internal__n_span_start_llhttp__on_body_2, s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete, + s_n_llhttp__internal__n_error_5, s_n_llhttp__internal__n_headers_almost_done, + s_n_llhttp__internal__n_header_field_colon_discard_ws, s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete, s_n_llhttp__internal__n_span_start_llhttp__on_header_value, s_n_llhttp__internal__n_header_value_discard_lws, s_n_llhttp__internal__n_header_value_discard_ws_almost_done, s_n_llhttp__internal__n_header_value_lws, s_n_llhttp__internal__n_header_value_almost_done, + s_n_llhttp__internal__n_invoke_test_lenient_flags_17, s_n_llhttp__internal__n_header_value_lenient, + s_n_llhttp__internal__n_error_54, s_n_llhttp__internal__n_header_value_otherwise, s_n_llhttp__internal__n_header_value_connection_token, s_n_llhttp__internal__n_header_value_connection_ws, @@ -364,17 +403,20 @@ enum llparse_state_e { s_n_llhttp__internal__n_header_value_connection_2, s_n_llhttp__internal__n_header_value_connection_3, s_n_llhttp__internal__n_header_value_connection, - s_n_llhttp__internal__n_error_23, - s_n_llhttp__internal__n_error_24, + s_n_llhttp__internal__n_error_56, + s_n_llhttp__internal__n_error_57, s_n_llhttp__internal__n_header_value_content_length_ws, s_n_llhttp__internal__n_header_value_content_length, - s_n_llhttp__internal__n_header_value_te_chunked_last, + s_n_llhttp__internal__n_error_59, + s_n_llhttp__internal__n_error_58, s_n_llhttp__internal__n_header_value_te_token_ows, s_n_llhttp__internal__n_header_value, s_n_llhttp__internal__n_header_value_te_token, + s_n_llhttp__internal__n_header_value_te_chunked_last, s_n_llhttp__internal__n_header_value_te_chunked, s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1, s_n_llhttp__internal__n_header_value_discard_ws, + s_n_llhttp__internal__n_invoke_load_header_state, s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete, s_n_llhttp__internal__n_header_field_general_otherwise, s_n_llhttp__internal__n_header_field_general, @@ -389,19 +431,37 @@ enum llparse_state_e { s_n_llhttp__internal__n_header_field, s_n_llhttp__internal__n_span_start_llhttp__on_header_field, s_n_llhttp__internal__n_header_field_start, + s_n_llhttp__internal__n_headers_start, s_n_llhttp__internal__n_url_to_http_09, s_n_llhttp__internal__n_url_skip_to_http09, s_n_llhttp__internal__n_url_skip_lf_to_http09_1, s_n_llhttp__internal__n_url_skip_lf_to_http09, s_n_llhttp__internal__n_req_pri_upgrade, - s_n_llhttp__internal__n_req_http_complete_1, + s_n_llhttp__internal__n_req_http_complete_crlf, s_n_llhttp__internal__n_req_http_complete, + s_n_llhttp__internal__n_invoke_load_method_1, + s_n_llhttp__internal__n_invoke_llhttp__on_version_complete, + s_n_llhttp__internal__n_error_67, + s_n_llhttp__internal__n_error_74, s_n_llhttp__internal__n_req_http_minor, + s_n_llhttp__internal__n_error_75, s_n_llhttp__internal__n_req_http_dot, + s_n_llhttp__internal__n_error_76, s_n_llhttp__internal__n_req_http_major, - s_n_llhttp__internal__n_req_http_start_1, - s_n_llhttp__internal__n_req_http_start_2, - s_n_llhttp__internal__n_req_http_start_3, + s_n_llhttp__internal__n_span_start_llhttp__on_version, + s_n_llhttp__internal__n_req_after_protocol, + s_n_llhttp__internal__n_invoke_load_method, + s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete, + s_n_llhttp__internal__n_error_82, + s_n_llhttp__internal__n_req_after_http_start_1, + s_n_llhttp__internal__n_invoke_load_method_2, + s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_1, + s_n_llhttp__internal__n_req_after_http_start_2, + s_n_llhttp__internal__n_invoke_load_method_3, + s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_2, + s_n_llhttp__internal__n_req_after_http_start_3, + s_n_llhttp__internal__n_req_after_http_start, + s_n_llhttp__internal__n_span_start_llhttp__on_protocol, s_n_llhttp__internal__n_req_http_start, s_n_llhttp__internal__n_url_to_http, s_n_llhttp__internal__n_url_skip_to_http, @@ -426,102 +486,135 @@ enum llparse_state_e { s_n_llhttp__internal__n_url_entry_connect, s_n_llhttp__internal__n_req_spaces_before_url, s_n_llhttp__internal__n_req_first_space_before_url, - s_n_llhttp__internal__n_start_req_2, - s_n_llhttp__internal__n_start_req_3, - s_n_llhttp__internal__n_start_req_1, - s_n_llhttp__internal__n_start_req_4, - s_n_llhttp__internal__n_start_req_6, - s_n_llhttp__internal__n_start_req_8, - s_n_llhttp__internal__n_start_req_9, - s_n_llhttp__internal__n_start_req_7, - s_n_llhttp__internal__n_start_req_5, - s_n_llhttp__internal__n_start_req_12, - s_n_llhttp__internal__n_start_req_13, - s_n_llhttp__internal__n_start_req_11, - s_n_llhttp__internal__n_start_req_10, - s_n_llhttp__internal__n_start_req_14, - s_n_llhttp__internal__n_start_req_17, - s_n_llhttp__internal__n_start_req_16, - s_n_llhttp__internal__n_start_req_15, - s_n_llhttp__internal__n_start_req_18, - s_n_llhttp__internal__n_start_req_20, - s_n_llhttp__internal__n_start_req_21, - s_n_llhttp__internal__n_start_req_19, - s_n_llhttp__internal__n_start_req_23, - s_n_llhttp__internal__n_start_req_24, - s_n_llhttp__internal__n_start_req_26, - s_n_llhttp__internal__n_start_req_28, - s_n_llhttp__internal__n_start_req_29, - s_n_llhttp__internal__n_start_req_27, - s_n_llhttp__internal__n_start_req_25, - s_n_llhttp__internal__n_start_req_30, - s_n_llhttp__internal__n_start_req_22, - s_n_llhttp__internal__n_start_req_31, - s_n_llhttp__internal__n_start_req_32, - s_n_llhttp__internal__n_start_req_35, - s_n_llhttp__internal__n_start_req_36, - s_n_llhttp__internal__n_start_req_34, - s_n_llhttp__internal__n_start_req_37, - s_n_llhttp__internal__n_start_req_38, - s_n_llhttp__internal__n_start_req_42, - s_n_llhttp__internal__n_start_req_43, - s_n_llhttp__internal__n_start_req_41, - s_n_llhttp__internal__n_start_req_40, - s_n_llhttp__internal__n_start_req_39, - s_n_llhttp__internal__n_start_req_45, - s_n_llhttp__internal__n_start_req_44, - s_n_llhttp__internal__n_start_req_33, - s_n_llhttp__internal__n_start_req_48, - s_n_llhttp__internal__n_start_req_49, - s_n_llhttp__internal__n_start_req_50, - s_n_llhttp__internal__n_start_req_51, - s_n_llhttp__internal__n_start_req_47, - s_n_llhttp__internal__n_start_req_46, - s_n_llhttp__internal__n_start_req_54, - s_n_llhttp__internal__n_start_req_56, - s_n_llhttp__internal__n_start_req_57, - s_n_llhttp__internal__n_start_req_55, - s_n_llhttp__internal__n_start_req_53, - s_n_llhttp__internal__n_start_req_58, - s_n_llhttp__internal__n_start_req_59, - s_n_llhttp__internal__n_start_req_52, - s_n_llhttp__internal__n_start_req_61, - s_n_llhttp__internal__n_start_req_62, - s_n_llhttp__internal__n_start_req_60, - s_n_llhttp__internal__n_start_req_65, - s_n_llhttp__internal__n_start_req_67, - s_n_llhttp__internal__n_start_req_68, - s_n_llhttp__internal__n_start_req_66, - s_n_llhttp__internal__n_start_req_69, - s_n_llhttp__internal__n_start_req_64, - s_n_llhttp__internal__n_start_req_63, - s_n_llhttp__internal__n_start_req, - s_n_llhttp__internal__n_invoke_llhttp__on_status_complete, + s_n_llhttp__internal__n_invoke_llhttp__on_method_complete_1, + s_n_llhttp__internal__n_after_start_req_2, + s_n_llhttp__internal__n_after_start_req_3, + s_n_llhttp__internal__n_after_start_req_1, + s_n_llhttp__internal__n_after_start_req_4, + s_n_llhttp__internal__n_after_start_req_6, + s_n_llhttp__internal__n_after_start_req_8, + s_n_llhttp__internal__n_after_start_req_9, + s_n_llhttp__internal__n_after_start_req_7, + s_n_llhttp__internal__n_after_start_req_5, + s_n_llhttp__internal__n_after_start_req_12, + s_n_llhttp__internal__n_after_start_req_13, + s_n_llhttp__internal__n_after_start_req_11, + s_n_llhttp__internal__n_after_start_req_10, + s_n_llhttp__internal__n_after_start_req_14, + s_n_llhttp__internal__n_after_start_req_17, + s_n_llhttp__internal__n_after_start_req_16, + s_n_llhttp__internal__n_after_start_req_15, + s_n_llhttp__internal__n_after_start_req_18, + s_n_llhttp__internal__n_after_start_req_20, + s_n_llhttp__internal__n_after_start_req_21, + s_n_llhttp__internal__n_after_start_req_19, + s_n_llhttp__internal__n_after_start_req_23, + s_n_llhttp__internal__n_after_start_req_24, + s_n_llhttp__internal__n_after_start_req_26, + s_n_llhttp__internal__n_after_start_req_28, + s_n_llhttp__internal__n_after_start_req_29, + s_n_llhttp__internal__n_after_start_req_27, + s_n_llhttp__internal__n_after_start_req_25, + s_n_llhttp__internal__n_after_start_req_30, + s_n_llhttp__internal__n_after_start_req_22, + s_n_llhttp__internal__n_after_start_req_31, + s_n_llhttp__internal__n_after_start_req_32, + s_n_llhttp__internal__n_after_start_req_35, + s_n_llhttp__internal__n_after_start_req_36, + s_n_llhttp__internal__n_after_start_req_34, + s_n_llhttp__internal__n_after_start_req_37, + s_n_llhttp__internal__n_after_start_req_38, + s_n_llhttp__internal__n_after_start_req_42, + s_n_llhttp__internal__n_after_start_req_43, + s_n_llhttp__internal__n_after_start_req_41, + s_n_llhttp__internal__n_after_start_req_40, + s_n_llhttp__internal__n_after_start_req_39, + s_n_llhttp__internal__n_after_start_req_45, + s_n_llhttp__internal__n_after_start_req_44, + s_n_llhttp__internal__n_after_start_req_33, + s_n_llhttp__internal__n_after_start_req_46, + s_n_llhttp__internal__n_after_start_req_49, + s_n_llhttp__internal__n_after_start_req_50, + s_n_llhttp__internal__n_after_start_req_51, + s_n_llhttp__internal__n_after_start_req_52, + s_n_llhttp__internal__n_after_start_req_48, + s_n_llhttp__internal__n_after_start_req_47, + s_n_llhttp__internal__n_after_start_req_55, + s_n_llhttp__internal__n_after_start_req_57, + s_n_llhttp__internal__n_after_start_req_58, + s_n_llhttp__internal__n_after_start_req_56, + s_n_llhttp__internal__n_after_start_req_54, + s_n_llhttp__internal__n_after_start_req_59, + s_n_llhttp__internal__n_after_start_req_60, + s_n_llhttp__internal__n_after_start_req_53, + s_n_llhttp__internal__n_after_start_req_62, + s_n_llhttp__internal__n_after_start_req_63, + s_n_llhttp__internal__n_after_start_req_61, + s_n_llhttp__internal__n_after_start_req_66, + s_n_llhttp__internal__n_after_start_req_68, + s_n_llhttp__internal__n_after_start_req_69, + s_n_llhttp__internal__n_after_start_req_67, + s_n_llhttp__internal__n_after_start_req_70, + s_n_llhttp__internal__n_after_start_req_65, + s_n_llhttp__internal__n_after_start_req_64, + s_n_llhttp__internal__n_after_start_req, + s_n_llhttp__internal__n_span_start_llhttp__on_method_1, s_n_llhttp__internal__n_res_line_almost_done, + s_n_llhttp__internal__n_invoke_test_lenient_flags_30, s_n_llhttp__internal__n_res_status, s_n_llhttp__internal__n_span_start_llhttp__on_status, - s_n_llhttp__internal__n_res_status_start, s_n_llhttp__internal__n_res_status_code_otherwise, - s_n_llhttp__internal__n_res_status_code, - s_n_llhttp__internal__n_res_http_end, + s_n_llhttp__internal__n_res_status_code_digit_3, + s_n_llhttp__internal__n_res_status_code_digit_2, + s_n_llhttp__internal__n_res_status_code_digit_1, + s_n_llhttp__internal__n_res_after_version, + s_n_llhttp__internal__n_invoke_llhttp__on_version_complete_1, + s_n_llhttp__internal__n_error_93, + s_n_llhttp__internal__n_error_107, s_n_llhttp__internal__n_res_http_minor, + s_n_llhttp__internal__n_error_108, s_n_llhttp__internal__n_res_http_dot, + s_n_llhttp__internal__n_error_109, s_n_llhttp__internal__n_res_http_major, - s_n_llhttp__internal__n_start_res, + s_n_llhttp__internal__n_span_start_llhttp__on_version_1, + s_n_llhttp__internal__n_res_after_protocol, + s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_3, + s_n_llhttp__internal__n_error_115, + s_n_llhttp__internal__n_res_after_start_1, + s_n_llhttp__internal__n_res_after_start_2, + s_n_llhttp__internal__n_res_after_start_3, + s_n_llhttp__internal__n_res_after_start, + s_n_llhttp__internal__n_span_start_llhttp__on_protocol_1, + s_n_llhttp__internal__n_invoke_llhttp__on_method_complete, s_n_llhttp__internal__n_req_or_res_method_2, + s_n_llhttp__internal__n_invoke_update_type_1, s_n_llhttp__internal__n_req_or_res_method_3, s_n_llhttp__internal__n_req_or_res_method_1, s_n_llhttp__internal__n_req_or_res_method, + s_n_llhttp__internal__n_span_start_llhttp__on_method, s_n_llhttp__internal__n_start_req_or_res, s_n_llhttp__internal__n_invoke_load_type, + s_n_llhttp__internal__n_invoke_update_finish, s_n_llhttp__internal__n_start, }; typedef enum llparse_state_e llparse_state_t; +int llhttp__on_method( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + int llhttp__on_url( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); +int llhttp__on_protocol( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_version( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + int llhttp__on_header_field( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); @@ -534,10 +627,29 @@ int llhttp__on_body( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); +int llhttp__on_chunk_extension_name( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_chunk_extension_value( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + int llhttp__on_status( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); +int llhttp__internal__c_load_initial_message_completed( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->initial_message_completed; +} + +int llhttp__on_reset( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + int llhttp__internal__c_update_finish( llhttp__internal_t* state, const unsigned char* p, @@ -566,6 +678,10 @@ int llhttp__internal__c_store_method( return 0; } +int llhttp__on_method_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + int llhttp__internal__c_is_equal_method( llhttp__internal_t* state, const unsigned char* p, @@ -593,6 +709,20 @@ int llhttp__on_url_complete( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); +int llhttp__internal__c_test_lenient_flags( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 1) == 1; +} + +int llhttp__internal__c_test_lenient_flags_1( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 256) == 256; +} + int llhttp__internal__c_test_flags( llhttp__internal_t* state, const unsigned char* p, @@ -619,6 +749,22 @@ int llhttp__after_message_complete( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); +int llhttp__internal__c_update_content_length( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->content_length = 0; + return 0; +} + +int llhttp__internal__c_update_initial_message_completed( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + state->initial_message_completed = 1; + return 0; +} + int llhttp__internal__c_update_finish_1( llhttp__internal_t* state, const unsigned char* p, @@ -627,25 +773,18 @@ int llhttp__internal__c_update_finish_1( return 0; } -int llhttp__internal__c_test_lenient_flags( +int llhttp__internal__c_test_lenient_flags_2( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { return (state->lenient_flags & 4) == 4; } -int llhttp__internal__c_test_flags_1( +int llhttp__internal__c_test_lenient_flags_3( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { - return (state->flags & 544) == 544; -} - -int llhttp__internal__c_test_lenient_flags_1( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return (state->lenient_flags & 2) == 2; + return (state->lenient_flags & 32) == 32; } int llhttp__before_headers_complete( @@ -660,14 +799,6 @@ int llhttp__after_headers_complete( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); -int llhttp__internal__c_update_content_length( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->content_length = 0; - return 0; -} - int llhttp__internal__c_mul_add_content_length( llhttp__internal_t* state, const unsigned char* p, @@ -677,9 +808,9 @@ int llhttp__internal__c_mul_add_content_length( if (state->content_length > 0xffffffffffffffffULL / 16) { return 1; } - + state->content_length *= 16; - + /* Addition overflow */ if (match >= 0) { if (state->content_length > 0xffffffffffffffffULL - match) { @@ -694,6 +825,13 @@ int llhttp__internal__c_mul_add_content_length( return 0; } +int llhttp__internal__c_test_lenient_flags_4( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 512) == 512; +} + int llhttp__on_chunk_header( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); @@ -705,6 +843,13 @@ int llhttp__internal__c_is_equal_content_length( return state->content_length == 0; } +int llhttp__internal__c_test_lenient_flags_7( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 128) == 128; +} + int llhttp__internal__c_or_flags( llhttp__internal_t* state, const unsigned char* p, @@ -713,6 +858,21 @@ int llhttp__internal__c_or_flags( return 0; } +int llhttp__internal__c_test_lenient_flags_8( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 64) == 64; +} + +int llhttp__on_chunk_extension_name_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__on_chunk_extension_value_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + int llhttp__internal__c_update_finish_3( llhttp__internal_t* state, const unsigned char* p, @@ -757,7 +917,21 @@ int llhttp__internal__c_load_header_state( return state->header_state; } -int llhttp__internal__c_or_flags_3( +int llhttp__internal__c_test_flags_4( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->flags & 512) == 512; +} + +int llhttp__internal__c_test_lenient_flags_22( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 2) == 2; +} + +int llhttp__internal__c_or_flags_5( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -777,7 +951,7 @@ int llhttp__on_header_value_complete( llhttp__internal_t* s, const unsigned char* p, const unsigned char* endp); -int llhttp__internal__c_or_flags_4( +int llhttp__internal__c_or_flags_6( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -785,7 +959,7 @@ int llhttp__internal__c_or_flags_4( return 0; } -int llhttp__internal__c_or_flags_5( +int llhttp__internal__c_or_flags_7( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -793,7 +967,7 @@ int llhttp__internal__c_or_flags_5( return 0; } -int llhttp__internal__c_or_flags_6( +int llhttp__internal__c_or_flags_8( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -801,7 +975,7 @@ int llhttp__internal__c_or_flags_6( return 0; } -int llhttp__internal__c_update_header_state_2( +int llhttp__internal__c_update_header_state_3( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -809,14 +983,7 @@ int llhttp__internal__c_update_header_state_2( return 0; } -int llhttp__internal__c_test_lenient_flags_2( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return (state->lenient_flags & 1) == 1; -} - -int llhttp__internal__c_update_header_state_4( +int llhttp__internal__c_update_header_state_1( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -824,7 +991,7 @@ int llhttp__internal__c_update_header_state_4( return 0; } -int llhttp__internal__c_update_header_state_5( +int llhttp__internal__c_update_header_state_6( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -832,7 +999,7 @@ int llhttp__internal__c_update_header_state_5( return 0; } -int llhttp__internal__c_update_header_state_6( +int llhttp__internal__c_update_header_state_7( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -856,9 +1023,9 @@ int llhttp__internal__c_mul_add_content_length_1( if (state->content_length > 0xffffffffffffffffULL / 10) { return 1; } - + state->content_length *= 10; - + /* Addition overflow */ if (match >= 0) { if (state->content_length > 0xffffffffffffffffULL - match) { @@ -873,7 +1040,7 @@ int llhttp__internal__c_mul_add_content_length_1( return 0; } -int llhttp__internal__c_or_flags_15( +int llhttp__internal__c_or_flags_17( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -881,7 +1048,21 @@ int llhttp__internal__c_or_flags_15( return 0; } -int llhttp__internal__c_or_flags_16( +int llhttp__internal__c_test_flags_3( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->flags & 8) == 8; +} + +int llhttp__internal__c_test_lenient_flags_20( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 8) == 8; +} + +int llhttp__internal__c_or_flags_18( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -897,7 +1078,7 @@ int llhttp__internal__c_and_flags( return 0; } -int llhttp__internal__c_update_header_state_7( +int llhttp__internal__c_update_header_state_8( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -905,7 +1086,7 @@ int llhttp__internal__c_update_header_state_7( return 0; } -int llhttp__internal__c_or_flags_17( +int llhttp__internal__c_or_flags_20( llhttp__internal_t* state, const unsigned char* p, const unsigned char* endp) { @@ -913,6 +1094,10 @@ int llhttp__internal__c_or_flags_17( return 0; } +int llhttp__on_protocol_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + int llhttp__internal__c_load_method( llhttp__internal_t* state, const unsigned char* p, @@ -938,6 +1123,31 @@ int llhttp__internal__c_store_http_minor( return 0; } +int llhttp__internal__c_test_lenient_flags_24( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return (state->lenient_flags & 16) == 16; +} + +int llhttp__on_version_complete( + llhttp__internal_t* s, const unsigned char* p, + const unsigned char* endp); + +int llhttp__internal__c_load_http_major( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->http_major; +} + +int llhttp__internal__c_load_http_minor( + llhttp__internal_t* state, + const unsigned char* p, + const unsigned char* endp) { + return state->http_minor; +} + int llhttp__internal__c_update_status_code( llhttp__internal_t* state, const unsigned char* p, @@ -955,9 +1165,9 @@ int llhttp__internal__c_mul_add_status_code( if (state->status_code > 0xffff / 10) { return 1; } - + state->status_code *= 10; - + /* Addition overflow */ if (match >= 0) { if (state->status_code > 0xffff - match) { @@ -969,11 +1179,6 @@ int llhttp__internal__c_mul_add_status_code( } } state->status_code += match; - - /* Enforce maximum */ - if (state->status_code > 999) { - return 1; - } return 0; } @@ -1025,22 +1230,20 @@ static llparse_state_t llhttp__internal__run( } default: { p++; - goto s_n_llhttp__internal__n_error_4; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_3; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_invoke_llhttp__after_message_complete: s_n_llhttp__internal__n_invoke_llhttp__after_message_complete: { switch (llhttp__after_message_complete(state, p, endp)) { case 1: - goto s_n_llhttp__internal__n_invoke_update_finish_2; + goto s_n_llhttp__internal__n_invoke_update_content_length; default: goto s_n_llhttp__internal__n_invoke_update_finish_1; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_pause_1: s_n_llhttp__internal__n_pause_1: { @@ -1049,8 +1252,7 @@ static llparse_state_t llhttp__internal__run( state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_invoke_is_equal_upgrade: s_n_llhttp__internal__n_invoke_is_equal_upgrade: { @@ -1060,8 +1262,7 @@ static llparse_state_t llhttp__internal__run( default: goto s_n_llhttp__internal__n_pause_1; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2: s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2: { @@ -1069,42 +1270,53 @@ static llparse_state_t llhttp__internal__run( case 0: goto s_n_llhttp__internal__n_invoke_is_equal_upgrade; case 21: - goto s_n_llhttp__internal__n_pause_5; + goto s_n_llhttp__internal__n_pause_13; default: - goto s_n_llhttp__internal__n_error_13; + goto s_n_llhttp__internal__n_error_38; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_chunk_data_almost_done: - s_n_llhttp__internal__n_chunk_data_almost_done: { - llparse_match_t match_seq; - + case s_n_llhttp__internal__n_chunk_data_almost_done_1: + s_n_llhttp__internal__n_chunk_data_almost_done_1: { if (p == endp) { - return s_n_llhttp__internal__n_chunk_data_almost_done; + return s_n_llhttp__internal__n_chunk_data_almost_done_1; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob0, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { + switch (*p) { + case 10: { p++; goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete; } - case kMatchPause: { - return s_n_llhttp__internal__n_chunk_data_almost_done; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_8; + default: { + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_7; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_chunk_data_almost_done: + s_n_llhttp__internal__n_chunk_data_almost_done: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_data_almost_done; + } + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_6; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_chunk_data_almost_done_1; + } + default: { + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_7; + } + } + UNREACHABLE; } case s_n_llhttp__internal__n_consume_content_length: s_n_llhttp__internal__n_consume_content_length: { size_t avail; - size_t need; - + uint64_t need; + avail = endp - p; need = state->content_length; if (avail >= need) { @@ -1112,11 +1324,10 @@ static llparse_state_t llhttp__internal__run( state->content_length = 0; goto s_n_llhttp__internal__n_span_end_llhttp__on_body; } - + state->content_length -= avail; return s_n_llhttp__internal__n_consume_content_length; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_start_llhttp__on_body: s_n_llhttp__internal__n_span_start_llhttp__on_body: { @@ -1126,8 +1337,7 @@ static llparse_state_t llhttp__internal__run( state->_span_pos0 = (void*) p; state->_span_cb0 = llhttp__on_body; goto s_n_llhttp__internal__n_consume_content_length; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_invoke_is_equal_content_length: s_n_llhttp__internal__n_invoke_is_equal_content_length: { @@ -1137,8 +1347,7 @@ static llparse_state_t llhttp__internal__run( default: goto s_n_llhttp__internal__n_invoke_or_flags; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_chunk_size_almost_done: s_n_llhttp__internal__n_chunk_size_almost_done: { @@ -1151,29 +1360,382 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_header; } default: { - goto s_n_llhttp__internal__n_error_9; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_8; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_chunk_parameters: - s_n_llhttp__internal__n_chunk_parameters: { + case s_n_llhttp__internal__n_invoke_test_lenient_flags_9: + s_n_llhttp__internal__n_invoke_test_lenient_flags_9: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_chunk_size_almost_done; + default: + goto s_n_llhttp__internal__n_error_20; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete: + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete: { + switch (llhttp__on_chunk_extension_name_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_9; + case 21: + goto s_n_llhttp__internal__n_pause_5; + default: + goto s_n_llhttp__internal__n_error_19; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_1: + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_1: { + switch (llhttp__on_chunk_extension_name_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_chunk_size_almost_done; + case 21: + goto s_n_llhttp__internal__n_pause_6; + default: + goto s_n_llhttp__internal__n_error_21; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_2: + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_2: { + switch (llhttp__on_chunk_extension_name_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_chunk_extensions; + case 21: + goto s_n_llhttp__internal__n_pause_7; + default: + goto s_n_llhttp__internal__n_error_22; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_test_lenient_flags_10: + s_n_llhttp__internal__n_invoke_test_lenient_flags_10: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_chunk_size_almost_done; + default: + goto s_n_llhttp__internal__n_error_25; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete: + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete: { + switch (llhttp__on_chunk_extension_value_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_10; + case 21: + goto s_n_llhttp__internal__n_pause_8; + default: + goto s_n_llhttp__internal__n_error_24; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1: + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1: { + switch (llhttp__on_chunk_extension_value_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_chunk_size_almost_done; + case 21: + goto s_n_llhttp__internal__n_pause_9; + default: + goto s_n_llhttp__internal__n_error_26; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_chunk_extension_quoted_value_done: + s_n_llhttp__internal__n_chunk_extension_quoted_value_done: { if (p == endp) { - return s_n_llhttp__internal__n_chunk_parameters; + return s_n_llhttp__internal__n_chunk_extension_quoted_value_done; } switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_11; + } case 13: { p++; goto s_n_llhttp__internal__n_chunk_size_almost_done; } - default: { + case ';': { p++; - goto s_n_llhttp__internal__n_chunk_parameters; + goto s_n_llhttp__internal__n_chunk_extensions; + } + default: { + goto s_n_llhttp__internal__n_error_29; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2: + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2: { + switch (llhttp__on_chunk_extension_value_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_chunk_extension_quoted_value_done; + case 21: + goto s_n_llhttp__internal__n_pause_10; + default: + goto s_n_llhttp__internal__n_error_27; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_30: + s_n_llhttp__internal__n_error_30: { + state->error = 0x2; + state->reason = "Invalid quoted-pair in chunk extensions quoted value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + case s_n_llhttp__internal__n_chunk_extension_quoted_value_quoted_pair: + s_n_llhttp__internal__n_chunk_extension_quoted_value_quoted_pair: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if (p == endp) { + return s_n_llhttp__internal__n_chunk_extension_quoted_value_quoted_pair; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_chunk_extension_quoted_value; + } + default: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_3; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_31: + s_n_llhttp__internal__n_error_31: { + state->error = 0x2; + state->reason = "Invalid character in chunk extensions quoted value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + case s_n_llhttp__internal__n_chunk_extension_quoted_value: + s_n_llhttp__internal__n_chunk_extension_quoted_value: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if (p == endp) { + return s_n_llhttp__internal__n_chunk_extension_quoted_value; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + p++; + goto s_n_llhttp__internal__n_chunk_extension_quoted_value; + } + case 2: { + p++; + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_2; + } + case 3: { + p++; + goto s_n_llhttp__internal__n_chunk_extension_quoted_value_quoted_pair; + } + default: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_4; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_3: + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_3: { + switch (llhttp__on_chunk_extension_value_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_chunk_extensions; + case 21: + goto s_n_llhttp__internal__n_pause_11; + default: + goto s_n_llhttp__internal__n_error_32; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_33: + s_n_llhttp__internal__n_error_33: { + state->error = 0x2; + state->reason = "Invalid character in chunk extensions value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + case s_n_llhttp__internal__n_chunk_extension_value: + s_n_llhttp__internal__n_chunk_extension_value: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 3, 4, 3, 3, 3, 3, 3, 0, 0, 3, 3, 0, 3, 3, 0, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 5, 0, 0, 0, 0, + 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 3, 0, 3, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_chunk_extension_value; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value; + } + case 2: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_1; + } + case 3: { + p++; + goto s_n_llhttp__internal__n_chunk_extension_value; + } + case 4: { + p++; + goto s_n_llhttp__internal__n_chunk_extension_quoted_value; + } + case 5: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_5; + } + default: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_6; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_value: + s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_value: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_value; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_chunk_extension_value; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_3; + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_34: + s_n_llhttp__internal__n_error_34: { + state->error = 0x2; + state->reason = "Invalid character in chunk extensions name"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + case s_n_llhttp__internal__n_chunk_extension_name: + s_n_llhttp__internal__n_chunk_extension_name: { + static uint8_t lookup_table[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 3, 0, 3, 3, 3, 3, 3, 0, 0, 3, 3, 0, 3, 3, 0, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 4, 0, 5, 0, 0, + 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 3, 0, 3, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + if (p == endp) { + return s_n_llhttp__internal__n_chunk_extension_name; + } + switch (lookup_table[(uint8_t) *p]) { + case 1: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name; + } + case 2: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_1; + } + case 3: { + p++; + goto s_n_llhttp__internal__n_chunk_extension_name; + } + case 4: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_2; + } + case 5: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_3; + } + default: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_4; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_name: + s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_name: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_name; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_chunk_extension_name; + goto s_n_llhttp__internal__n_chunk_extension_name; + UNREACHABLE; + } + case s_n_llhttp__internal__n_chunk_extensions: + s_n_llhttp__internal__n_chunk_extensions: { + if (p == endp) { + return s_n_llhttp__internal__n_chunk_extensions; + } + switch (*p) { + case 13: { + p++; + goto s_n_llhttp__internal__n_error_17; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_error_18; + } + default: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_name; + } + } + UNREACHABLE; } case s_n_llhttp__internal__n_chunk_size_otherwise: s_n_llhttp__internal__n_chunk_size_otherwise: { @@ -1181,24 +1743,31 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_chunk_size_otherwise; } switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_4; + } + case 10: { + p++; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_5; + } case 13: { p++; goto s_n_llhttp__internal__n_chunk_size_almost_done; } case ' ': { p++; - goto s_n_llhttp__internal__n_chunk_parameters; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_4; } case ';': { p++; - goto s_n_llhttp__internal__n_chunk_parameters; + goto s_n_llhttp__internal__n_chunk_extensions; } default: { - goto s_n_llhttp__internal__n_error_10; + goto s_n_llhttp__internal__n_error_35; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_chunk_size: s_n_llhttp__internal__n_chunk_size: { @@ -1320,8 +1889,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_chunk_size_otherwise; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_chunk_size_digit: s_n_llhttp__internal__n_chunk_size_digit: { @@ -1440,26 +2008,24 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_mul_add_content_length; } default: { - goto s_n_llhttp__internal__n_error_12; + goto s_n_llhttp__internal__n_error_37; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_invoke_update_content_length: - s_n_llhttp__internal__n_invoke_update_content_length: { + case s_n_llhttp__internal__n_invoke_update_content_length_1: + s_n_llhttp__internal__n_invoke_update_content_length_1: { switch (llhttp__internal__c_update_content_length(state, p, endp)) { default: goto s_n_llhttp__internal__n_chunk_size_digit; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_consume_content_length_1: s_n_llhttp__internal__n_consume_content_length_1: { size_t avail; - size_t need; - + uint64_t need; + avail = endp - p; need = state->content_length; if (avail >= need) { @@ -1467,11 +2033,10 @@ static llparse_state_t llhttp__internal__run( state->content_length = 0; goto s_n_llhttp__internal__n_span_end_llhttp__on_body_1; } - + state->content_length -= avail; return s_n_llhttp__internal__n_consume_content_length_1; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_start_llhttp__on_body_1: s_n_llhttp__internal__n_span_start_llhttp__on_body_1: { @@ -1481,8 +2046,7 @@ static llparse_state_t llhttp__internal__run( state->_span_pos0 = (void*) p; state->_span_cb0 = llhttp__on_body; goto s_n_llhttp__internal__n_consume_content_length_1; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_eof: s_n_llhttp__internal__n_eof: { @@ -1491,8 +2055,7 @@ static llparse_state_t llhttp__internal__run( } p++; goto s_n_llhttp__internal__n_eof; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_start_llhttp__on_body_2: s_n_llhttp__internal__n_span_start_llhttp__on_body_2: { @@ -1502,8 +2065,7 @@ static llparse_state_t llhttp__internal__run( state->_span_pos0 = (void*) p; state->_span_cb0 = llhttp__on_body; goto s_n_llhttp__internal__n_eof; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete: s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete: { @@ -1511,18 +2073,26 @@ static llparse_state_t llhttp__internal__run( case 1: goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_1; case 2: - goto s_n_llhttp__internal__n_invoke_update_content_length; + goto s_n_llhttp__internal__n_invoke_update_content_length_1; case 3: goto s_n_llhttp__internal__n_span_start_llhttp__on_body_1; case 4: goto s_n_llhttp__internal__n_invoke_update_finish_3; case 5: - goto s_n_llhttp__internal__n_error_14; + goto s_n_llhttp__internal__n_error_39; default: goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_5: + s_n_llhttp__internal__n_error_5: { + state->error = 0xa; + state->reason = "Invalid header field char"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } case s_n_llhttp__internal__n_headers_almost_done: s_n_llhttp__internal__n_headers_almost_done: { @@ -1532,23 +2102,41 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 10: { p++; - goto s_n_llhttp__internal__n_invoke_test_flags; + goto s_n_llhttp__internal__n_invoke_test_flags_1; } default: { - goto s_n_llhttp__internal__n_error_17; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_12; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_header_field_colon_discard_ws: + s_n_llhttp__internal__n_header_field_colon_discard_ws: { + if (p == endp) { + return s_n_llhttp__internal__n_header_field_colon_discard_ws; + } + switch (*p) { + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_field_colon_discard_ws; + } + default: { + goto s_n_llhttp__internal__n_header_field_colon; + } + } + UNREACHABLE; } case s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete: s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete: { switch (llhttp__on_header_value_complete(state, p, endp)) { - default: + case 0: goto s_n_llhttp__internal__n_header_field_start; + case 21: + goto s_n_llhttp__internal__n_pause_18; + default: + goto s_n_llhttp__internal__n_error_48; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_start_llhttp__on_header_value: s_n_llhttp__internal__n_span_start_llhttp__on_header_value: { @@ -1558,8 +2146,7 @@ static llparse_state_t llhttp__internal__run( state->_span_pos0 = (void*) p; state->_span_cb0 = llhttp__on_header_value; goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_discard_lws: s_n_llhttp__internal__n_header_value_discard_lws: { @@ -1569,18 +2156,17 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 9: { p++; - goto s_n_llhttp__internal__n_header_value_discard_ws; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_15; } case ' ': { p++; - goto s_n_llhttp__internal__n_header_value_discard_ws; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_15; } default: { - goto s_n_llhttp__internal__n_invoke_load_header_state; + goto s_n_llhttp__internal__n_invoke_load_header_state_1; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_discard_ws_almost_done: s_n_llhttp__internal__n_header_value_discard_ws_almost_done: { @@ -1593,11 +2179,10 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_discard_lws; } default: { - goto s_n_llhttp__internal__n_error_19; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_16; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_lws: s_n_llhttp__internal__n_header_value_lws: { @@ -1606,17 +2191,16 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case 9: { - goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_18; } case ' ': { - goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_18; } default: { - goto s_n_llhttp__internal__n_invoke_load_header_state_3; + goto s_n_llhttp__internal__n_invoke_load_header_state_5; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_almost_done: s_n_llhttp__internal__n_header_value_almost_done: { @@ -1629,11 +2213,20 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_lws; } default: { - goto s_n_llhttp__internal__n_error_20; + goto s_n_llhttp__internal__n_error_53; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_test_lenient_flags_17: + s_n_llhttp__internal__n_invoke_test_lenient_flags_17: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_header_value_almost_done; + default: + goto s_n_llhttp__internal__n_error_51; + } + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_lenient: s_n_llhttp__internal__n_header_value_lenient: { @@ -1642,18 +2235,26 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case 10: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4; } case 13: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5; } default: { p++; goto s_n_llhttp__internal__n_header_value_lenient; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_54: + s_n_llhttp__internal__n_error_54: { + state->error = 0xa; + state->reason = "Invalid header value char"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_otherwise: s_n_llhttp__internal__n_header_value_otherwise: { @@ -1668,11 +2269,10 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2; } default: { - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_2; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_19; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_connection_token: s_n_llhttp__internal__n_header_value_connection_token: { @@ -1710,8 +2310,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_otherwise; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_connection_ws: s_n_llhttp__internal__n_header_value_connection_ws: { @@ -1731,28 +2330,27 @@ static llparse_state_t llhttp__internal__run( } case ',': { p++; - goto s_n_llhttp__internal__n_invoke_load_header_state_4; + goto s_n_llhttp__internal__n_invoke_load_header_state_6; } default: { - goto s_n_llhttp__internal__n_invoke_update_header_state_4; + goto s_n_llhttp__internal__n_invoke_update_header_state_5; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_connection_1: s_n_llhttp__internal__n_header_value_connection_1: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_header_value_connection_1; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob3, 4); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob2, 4); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { p++; - goto s_n_llhttp__internal__n_invoke_update_header_state_2; + goto s_n_llhttp__internal__n_invoke_update_header_state_3; } case kMatchPause: { return s_n_llhttp__internal__n_header_value_connection_1; @@ -1761,22 +2359,21 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_connection_token; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_connection_2: s_n_llhttp__internal__n_header_value_connection_2: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_header_value_connection_2; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob4, 9); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob3, 9); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { p++; - goto s_n_llhttp__internal__n_invoke_update_header_state_5; + goto s_n_llhttp__internal__n_invoke_update_header_state_6; } case kMatchPause: { return s_n_llhttp__internal__n_header_value_connection_2; @@ -1785,22 +2382,21 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_connection_token; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_connection_3: s_n_llhttp__internal__n_header_value_connection_3: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_header_value_connection_3; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob5, 6); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob4, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { p++; - goto s_n_llhttp__internal__n_invoke_update_header_state_6; + goto s_n_llhttp__internal__n_invoke_update_header_state_7; } case kMatchPause: { return s_n_llhttp__internal__n_header_value_connection_3; @@ -1809,8 +2405,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_connection_token; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_connection: s_n_llhttp__internal__n_header_value_connection: { @@ -1842,28 +2437,25 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_connection_token; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_error_23: - s_n_llhttp__internal__n_error_23: { + case s_n_llhttp__internal__n_error_56: + s_n_llhttp__internal__n_error_56: { state->error = 0xb; state->reason = "Content-Length overflow"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_error_24: - s_n_llhttp__internal__n_error_24: { + case s_n_llhttp__internal__n_error_57: + s_n_llhttp__internal__n_error_57: { state->error = 0xb; state->reason = "Invalid character in Content-Length"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_content_length_ws: s_n_llhttp__internal__n_header_value_content_length_ws: { @@ -1872,21 +2464,20 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case 10: { - goto s_n_llhttp__internal__n_invoke_or_flags_15; + goto s_n_llhttp__internal__n_invoke_or_flags_17; } case 13: { - goto s_n_llhttp__internal__n_invoke_or_flags_15; + goto s_n_llhttp__internal__n_invoke_or_flags_17; } case ' ': { p++; goto s_n_llhttp__internal__n_header_value_content_length_ws; } default: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_7; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_content_length: s_n_llhttp__internal__n_header_value_content_length: { @@ -1948,31 +2539,25 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_content_length_ws; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_header_value_te_chunked_last: - s_n_llhttp__internal__n_header_value_te_chunked_last: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_te_chunked_last; - } - switch (*p) { - case 10: { - goto s_n_llhttp__internal__n_invoke_update_header_state_7; - } - case 13: { - goto s_n_llhttp__internal__n_invoke_update_header_state_7; - } - case ' ': { - p++; - goto s_n_llhttp__internal__n_header_value_te_chunked_last; - } - default: { - goto s_n_llhttp__internal__n_header_value_te_chunked; - } - } - /* UNREACHABLE */; - abort(); + case s_n_llhttp__internal__n_error_59: + s_n_llhttp__internal__n_error_59: { + state->error = 0xf; + state->reason = "Invalid `Transfer-Encoding` header value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_58: + s_n_llhttp__internal__n_error_58: { + state->error = 0xf; + state->reason = "Invalid `Transfer-Encoding` header value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_te_token_ows: s_n_llhttp__internal__n_header_value_te_token_ows: { @@ -1992,8 +2577,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_te_chunked; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value: s_n_llhttp__internal__n_header_value: { @@ -2022,19 +2606,18 @@ static llparse_state_t llhttp__internal__run( if (endp - p >= 16) { __m128i ranges; __m128i input; - int avail; int match_len; - + /* Load input */ input = _mm_loadu_si128((__m128i const*) p); - ranges = _mm_loadu_si128((__m128i const*) llparse_blob7); - + ranges = _mm_loadu_si128((__m128i const*) llparse_blob6); + /* Find first character that does not match `ranges` */ match_len = _mm_cmpestri(ranges, 6, input, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | _SIDD_NEGATIVE_POLARITY); - + if (match_len != 0) { p += match_len; goto s_n_llhttp__internal__n_header_value; @@ -2042,6 +2625,78 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_otherwise; } #endif /* __SSE4_2__ */ + #ifdef __ARM_NEON__ + while (endp - p >= 16) { + uint8x16_t input; + uint8x16_t single; + uint8x16_t mask; + uint8x8_t narrow; + uint64_t match_mask; + int match_len; + + /* Load input */ + input = vld1q_u8(p); + /* Find first character that does not match `ranges` */ + single = vceqq_u8(input, vdupq_n_u8(0x9)); + mask = single; + single = vandq_u16( + vcgeq_u8(input, vdupq_n_u8(' ')), + vcleq_u8(input, vdupq_n_u8('~')) + ); + mask = vorrq_u16(mask, single); + single = vandq_u16( + vcgeq_u8(input, vdupq_n_u8(0x80)), + vcleq_u8(input, vdupq_n_u8(0xff)) + ); + mask = vorrq_u16(mask, single); + narrow = vshrn_n_u16(mask, 4); + match_mask = ~vget_lane_u64(vreinterpret_u64_u8(narrow), 0); + match_len = __builtin_ctzll(match_mask) >> 2; + if (match_len != 16) { + p += match_len; + goto s_n_llhttp__internal__n_header_value_otherwise; + } + p += 16; + } + if (p == endp) { + return s_n_llhttp__internal__n_header_value; + } + #endif /* __ARM_NEON__ */ + #ifdef __wasm_simd128__ + while (endp - p >= 16) { + v128_t input; + v128_t mask; + v128_t single; + int match_len; + + /* Load input */ + input = wasm_v128_load(p); + /* Find first character that does not match `ranges` */ + single = wasm_i8x16_eq(input, wasm_u8x16_const_splat(0x9)); + mask = single; + single = wasm_v128_and( + wasm_i8x16_ge(input, wasm_u8x16_const_splat(' ')), + wasm_i8x16_le(input, wasm_u8x16_const_splat('~')) + ); + mask = wasm_v128_or(mask, single); + single = wasm_v128_and( + wasm_i8x16_ge(input, wasm_u8x16_const_splat(0x80)), + wasm_i8x16_le(input, wasm_u8x16_const_splat(0xff)) + ); + mask = wasm_v128_or(mask, single); + match_len = __builtin_ctz( + ~wasm_i8x16_bitmask(mask) + ); + if (match_len != 16) { + p += match_len; + goto s_n_llhttp__internal__n_header_value_otherwise; + } + p += 16; + } + if (p == endp) { + return s_n_llhttp__internal__n_header_value; + } + #endif /* __wasm_simd128__ */ switch (lookup_table[(uint8_t) *p]) { case 1: { p++; @@ -2051,8 +2706,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_otherwise; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_te_token: s_n_llhttp__internal__n_header_value_te_token: { @@ -2087,20 +2741,44 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_te_token_ows; } default: { - goto s_n_llhttp__internal__n_invoke_update_header_state_8; + goto s_n_llhttp__internal__n_invoke_update_header_state_9; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_header_value_te_chunked_last: + s_n_llhttp__internal__n_header_value_te_chunked_last: { + if (p == endp) { + return s_n_llhttp__internal__n_header_value_te_chunked_last; + } + switch (*p) { + case 10: { + goto s_n_llhttp__internal__n_invoke_update_header_state_8; + } + case 13: { + goto s_n_llhttp__internal__n_invoke_update_header_state_8; + } + case ' ': { + p++; + goto s_n_llhttp__internal__n_header_value_te_chunked_last; + } + case ',': { + goto s_n_llhttp__internal__n_invoke_load_type_1; + } + default: { + goto s_n_llhttp__internal__n_header_value_te_token; + } + } + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_te_chunked: s_n_llhttp__internal__n_header_value_te_chunked: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_header_value_te_chunked; } - match_seq = llparse__match_sequence_to_lower_unsafe(state, p, endp, llparse_blob6, 7); + match_seq = llparse__match_sequence_to_lower_unsafe(state, p, endp, llparse_blob5, 7); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2114,8 +2792,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_value_te_token; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1: s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1: { @@ -2124,9 +2801,8 @@ static llparse_state_t llhttp__internal__run( } state->_span_pos0 = (void*) p; state->_span_cb0 = llhttp__on_header_value; - goto s_n_llhttp__internal__n_invoke_load_header_state_2; - /* UNREACHABLE */; - abort(); + goto s_n_llhttp__internal__n_invoke_load_header_state_3; + UNREACHABLE; } case s_n_llhttp__internal__n_header_value_discard_ws: s_n_llhttp__internal__n_header_value_discard_ws: { @@ -2140,7 +2816,7 @@ static llparse_state_t llhttp__internal__run( } case 10: { p++; - goto s_n_llhttp__internal__n_header_value_discard_lws; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_14; } case 13: { p++; @@ -2154,17 +2830,31 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_load_header_state: + s_n_llhttp__internal__n_invoke_load_header_state: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 2: + goto s_n_llhttp__internal__n_invoke_test_flags_4; + case 3: + goto s_n_llhttp__internal__n_invoke_test_flags_5; + default: + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + UNREACHABLE; } case s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete: s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete: { switch (llhttp__on_header_field_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_load_header_state; + case 21: + goto s_n_llhttp__internal__n_pause_19; default: - goto s_n_llhttp__internal__n_header_value_discard_ws; + goto s_n_llhttp__internal__n_error_45; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_field_general_otherwise: s_n_llhttp__internal__n_header_field_general_otherwise: { @@ -2173,14 +2863,13 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case ':': { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field_1; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field_2; } default: { - goto s_n_llhttp__internal__n_error_25; + goto s_n_llhttp__internal__n_error_62; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_field_general: s_n_llhttp__internal__n_header_field_general: { @@ -2209,31 +2898,30 @@ static llparse_state_t llhttp__internal__run( if (endp - p >= 16) { __m128i ranges; __m128i input; - int avail; int match_len; - + /* Load input */ input = _mm_loadu_si128((__m128i const*) p); - ranges = _mm_loadu_si128((__m128i const*) llparse_blob8); - + ranges = _mm_loadu_si128((__m128i const*) llparse_blob7); + /* Find first character that does not match `ranges` */ match_len = _mm_cmpestri(ranges, 16, input, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | _SIDD_NEGATIVE_POLARITY); - + if (match_len != 0) { p += match_len; goto s_n_llhttp__internal__n_header_field_general; } - ranges = _mm_loadu_si128((__m128i const*) llparse_blob9); - + ranges = _mm_loadu_si128((__m128i const*) llparse_blob8); + /* Find first character that does not match `ranges` */ match_len = _mm_cmpestri(ranges, 2, input, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | _SIDD_NEGATIVE_POLARITY); - + if (match_len != 0) { p += match_len; goto s_n_llhttp__internal__n_header_field_general; @@ -2250,8 +2938,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_field_general_otherwise; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_field_colon: s_n_llhttp__internal__n_header_field_colon: { @@ -2260,27 +2947,25 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case ' ': { - p++; - goto s_n_llhttp__internal__n_header_field_colon; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_13; } case ':': { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field_1; } default: { - goto s_n_llhttp__internal__n_invoke_update_header_state_9; + goto s_n_llhttp__internal__n_invoke_update_header_state_10; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_field_3: s_n_llhttp__internal__n_header_field_3: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_header_field_3; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob2, 6); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob1, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2292,20 +2977,19 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_header_field_3; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; + goto s_n_llhttp__internal__n_invoke_update_header_state_11; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_field_4: s_n_llhttp__internal__n_header_field_4: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_header_field_4; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob10, 10); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob9, 10); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2317,11 +3001,10 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_header_field_4; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; + goto s_n_llhttp__internal__n_invoke_update_header_state_11; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_field_2: s_n_llhttp__internal__n_header_field_2: { @@ -2338,20 +3021,19 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_field_4; } default: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; + goto s_n_llhttp__internal__n_invoke_update_header_state_11; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_field_1: s_n_llhttp__internal__n_header_field_1: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_header_field_1; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob1, 2); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob0, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2362,20 +3044,19 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_header_field_1; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; + goto s_n_llhttp__internal__n_invoke_update_header_state_11; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_field_5: s_n_llhttp__internal__n_header_field_5: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_header_field_5; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob11, 15); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob10, 15); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2387,20 +3068,19 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_header_field_5; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; + goto s_n_llhttp__internal__n_invoke_update_header_state_11; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_field_6: s_n_llhttp__internal__n_header_field_6: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_header_field_6; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob12, 16); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob11, 16); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2412,20 +3092,19 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_header_field_6; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; + goto s_n_llhttp__internal__n_invoke_update_header_state_11; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_field_7: s_n_llhttp__internal__n_header_field_7: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_header_field_7; } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob13, 6); + match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob12, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -2437,11 +3116,10 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_header_field_7; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; + goto s_n_llhttp__internal__n_invoke_update_header_state_11; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_field: s_n_llhttp__internal__n_header_field: { @@ -2466,11 +3144,10 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_header_field_7; } default: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; + goto s_n_llhttp__internal__n_invoke_update_header_state_11; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_start_llhttp__on_header_field: s_n_llhttp__internal__n_span_start_llhttp__on_header_field: { @@ -2480,8 +3157,7 @@ static llparse_state_t llhttp__internal__run( state->_span_pos0 = (void*) p; state->_span_cb0 = llhttp__on_header_field; goto s_n_llhttp__internal__n_header_field; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_header_field_start: s_n_llhttp__internal__n_header_field_start: { @@ -2490,18 +3166,37 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case 10: { - goto s_n_llhttp__internal__n_headers_almost_done; + p++; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_1; } case 13: { p++; goto s_n_llhttp__internal__n_headers_almost_done; } + case ':': { + goto s_n_llhttp__internal__n_error_44; + } default: { goto s_n_llhttp__internal__n_span_start_llhttp__on_header_field; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_headers_start: + s_n_llhttp__internal__n_headers_start: { + if (p == endp) { + return s_n_llhttp__internal__n_headers_start; + } + switch (*p) { + case ' ': { + p++; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags; + } + default: { + goto s_n_llhttp__internal__n_header_field_start; + } + } + UNREACHABLE; } case s_n_llhttp__internal__n_url_to_http_09: s_n_llhttp__internal__n_url_to_http_09: { @@ -2511,18 +3206,17 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 9: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 12: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } default: { goto s_n_llhttp__internal__n_invoke_update_http_major; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_skip_to_http09: s_n_llhttp__internal__n_url_skip_to_http09: { @@ -2532,19 +3226,18 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 9: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 12: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } default: { p++; goto s_n_llhttp__internal__n_url_to_http_09; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_skip_lf_to_http09_1: s_n_llhttp__internal__n_url_skip_lf_to_http09_1: { @@ -2557,11 +3250,10 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_to_http_09; } default: { - goto s_n_llhttp__internal__n_error_26; + goto s_n_llhttp__internal__n_error_63; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_skip_lf_to_http09: s_n_llhttp__internal__n_url_skip_lf_to_http09: { @@ -2571,63 +3263,60 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 9: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 12: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 13: { p++; goto s_n_llhttp__internal__n_url_skip_lf_to_http09_1; } default: { - goto s_n_llhttp__internal__n_error_26; + goto s_n_llhttp__internal__n_error_63; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_req_pri_upgrade: s_n_llhttp__internal__n_req_pri_upgrade: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_req_pri_upgrade; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob15, 10); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob14, 10); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { p++; - goto s_n_llhttp__internal__n_error_29; + goto s_n_llhttp__internal__n_error_72; } case kMatchPause: { return s_n_llhttp__internal__n_req_pri_upgrade; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_30; + goto s_n_llhttp__internal__n_error_73; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_req_http_complete_1: - s_n_llhttp__internal__n_req_http_complete_1: { + case s_n_llhttp__internal__n_req_http_complete_crlf: + s_n_llhttp__internal__n_req_http_complete_crlf: { if (p == endp) { - return s_n_llhttp__internal__n_req_http_complete_1; + return s_n_llhttp__internal__n_req_http_complete_crlf; } switch (*p) { case 10: { p++; - goto s_n_llhttp__internal__n_header_field_start; + goto s_n_llhttp__internal__n_headers_start; } default: { - goto s_n_llhttp__internal__n_error_28; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_26; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_req_http_complete: s_n_llhttp__internal__n_req_http_complete: { @@ -2637,18 +3326,57 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 10: { p++; - goto s_n_llhttp__internal__n_header_field_start; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_25; } case 13: { p++; - goto s_n_llhttp__internal__n_req_http_complete_1; + goto s_n_llhttp__internal__n_req_http_complete_crlf; } default: { - goto s_n_llhttp__internal__n_error_28; + goto s_n_llhttp__internal__n_error_71; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_load_method_1: + s_n_llhttp__internal__n_invoke_load_method_1: { + switch (llhttp__internal__c_load_method(state, p, endp)) { + case 34: + goto s_n_llhttp__internal__n_req_pri_upgrade; + default: + goto s_n_llhttp__internal__n_req_http_complete; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_version_complete: + s_n_llhttp__internal__n_invoke_llhttp__on_version_complete: { + switch (llhttp__on_version_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_load_method_1; + case 21: + goto s_n_llhttp__internal__n_pause_21; + default: + goto s_n_llhttp__internal__n_error_68; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_67: + s_n_llhttp__internal__n_error_67: { + state->error = 0x9; + state->reason = "Invalid HTTP version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_74: + s_n_llhttp__internal__n_error_74: { + state->error = 0x9; + state->reason = "Invalid minor version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } case s_n_llhttp__internal__n_req_http_minor: s_n_llhttp__internal__n_req_http_minor: { @@ -2707,11 +3435,19 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_http_minor; } default: { - goto s_n_llhttp__internal__n_error_31; + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_2; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_75: + s_n_llhttp__internal__n_error_75: { + state->error = 0x9; + state->reason = "Expected dot"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } case s_n_llhttp__internal__n_req_http_dot: s_n_llhttp__internal__n_req_http_dot: { @@ -2724,11 +3460,19 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_req_http_minor; } default: { - goto s_n_llhttp__internal__n_error_32; + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_3; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_76: + s_n_llhttp__internal__n_error_76: { + state->error = 0x9; + state->reason = "Invalid major version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } case s_n_llhttp__internal__n_req_http_major: s_n_llhttp__internal__n_req_http_major: { @@ -2787,83 +3531,310 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_http_major; } default: { - goto s_n_llhttp__internal__n_error_33; + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_4; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_req_http_start_1: - s_n_llhttp__internal__n_req_http_start_1: { - llparse_match_t match_seq; - + case s_n_llhttp__internal__n_span_start_llhttp__on_version: + s_n_llhttp__internal__n_span_start_llhttp__on_version: { if (p == endp) { - return s_n_llhttp__internal__n_req_http_start_1; + return s_n_llhttp__internal__n_span_start_llhttp__on_version; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob14, 4); + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_version; + goto s_n_llhttp__internal__n_req_http_major; + UNREACHABLE; + } + case s_n_llhttp__internal__n_req_after_protocol: + s_n_llhttp__internal__n_req_after_protocol: { + if (p == endp) { + return s_n_llhttp__internal__n_req_after_protocol; + } + switch (*p) { + case '/': { + p++; + goto s_n_llhttp__internal__n_span_start_llhttp__on_version; + } + default: { + goto s_n_llhttp__internal__n_error_77; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_load_method: + s_n_llhttp__internal__n_invoke_load_method: { + switch (llhttp__internal__c_load_method(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_req_after_protocol; + case 1: + goto s_n_llhttp__internal__n_req_after_protocol; + case 2: + goto s_n_llhttp__internal__n_req_after_protocol; + case 3: + goto s_n_llhttp__internal__n_req_after_protocol; + case 4: + goto s_n_llhttp__internal__n_req_after_protocol; + case 5: + goto s_n_llhttp__internal__n_req_after_protocol; + case 6: + goto s_n_llhttp__internal__n_req_after_protocol; + case 7: + goto s_n_llhttp__internal__n_req_after_protocol; + case 8: + goto s_n_llhttp__internal__n_req_after_protocol; + case 9: + goto s_n_llhttp__internal__n_req_after_protocol; + case 10: + goto s_n_llhttp__internal__n_req_after_protocol; + case 11: + goto s_n_llhttp__internal__n_req_after_protocol; + case 12: + goto s_n_llhttp__internal__n_req_after_protocol; + case 13: + goto s_n_llhttp__internal__n_req_after_protocol; + case 14: + goto s_n_llhttp__internal__n_req_after_protocol; + case 15: + goto s_n_llhttp__internal__n_req_after_protocol; + case 16: + goto s_n_llhttp__internal__n_req_after_protocol; + case 17: + goto s_n_llhttp__internal__n_req_after_protocol; + case 18: + goto s_n_llhttp__internal__n_req_after_protocol; + case 19: + goto s_n_llhttp__internal__n_req_after_protocol; + case 20: + goto s_n_llhttp__internal__n_req_after_protocol; + case 21: + goto s_n_llhttp__internal__n_req_after_protocol; + case 22: + goto s_n_llhttp__internal__n_req_after_protocol; + case 23: + goto s_n_llhttp__internal__n_req_after_protocol; + case 24: + goto s_n_llhttp__internal__n_req_after_protocol; + case 25: + goto s_n_llhttp__internal__n_req_after_protocol; + case 26: + goto s_n_llhttp__internal__n_req_after_protocol; + case 27: + goto s_n_llhttp__internal__n_req_after_protocol; + case 28: + goto s_n_llhttp__internal__n_req_after_protocol; + case 29: + goto s_n_llhttp__internal__n_req_after_protocol; + case 30: + goto s_n_llhttp__internal__n_req_after_protocol; + case 31: + goto s_n_llhttp__internal__n_req_after_protocol; + case 32: + goto s_n_llhttp__internal__n_req_after_protocol; + case 33: + goto s_n_llhttp__internal__n_req_after_protocol; + case 34: + goto s_n_llhttp__internal__n_req_after_protocol; + case 46: + goto s_n_llhttp__internal__n_req_after_protocol; + default: + goto s_n_llhttp__internal__n_error_66; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete: + s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete: { + switch (llhttp__on_protocol_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_load_method; + case 21: + goto s_n_llhttp__internal__n_pause_22; + default: + goto s_n_llhttp__internal__n_error_65; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_82: + s_n_llhttp__internal__n_error_82: { + state->error = 0x8; + state->reason = "Expected HTTP/, RTSP/ or ICE/"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + case s_n_llhttp__internal__n_req_after_http_start_1: + s_n_llhttp__internal__n_req_after_http_start_1: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_after_http_start_1; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob13, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { p++; - goto s_n_llhttp__internal__n_invoke_load_method; + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol; } case kMatchPause: { - return s_n_llhttp__internal__n_req_http_start_1; + return s_n_llhttp__internal__n_req_after_http_start_1; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_36; + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_3; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_req_http_start_2: - s_n_llhttp__internal__n_req_http_start_2: { + case s_n_llhttp__internal__n_invoke_load_method_2: + s_n_llhttp__internal__n_invoke_load_method_2: { + switch (llhttp__internal__c_load_method(state, p, endp)) { + case 33: + goto s_n_llhttp__internal__n_req_after_protocol; + default: + goto s_n_llhttp__internal__n_error_79; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_1: + s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_1: { + switch (llhttp__on_protocol_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_load_method_2; + case 21: + goto s_n_llhttp__internal__n_pause_23; + default: + goto s_n_llhttp__internal__n_error_78; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_req_after_http_start_2: + s_n_llhttp__internal__n_req_after_http_start_2: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_req_http_start_2; + return s_n_llhttp__internal__n_req_after_http_start_2; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob15, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_req_after_http_start_2; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_3; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_load_method_3: + s_n_llhttp__internal__n_invoke_load_method_3: { + switch (llhttp__internal__c_load_method(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_req_after_protocol; + case 3: + goto s_n_llhttp__internal__n_req_after_protocol; + case 6: + goto s_n_llhttp__internal__n_req_after_protocol; + case 35: + goto s_n_llhttp__internal__n_req_after_protocol; + case 36: + goto s_n_llhttp__internal__n_req_after_protocol; + case 37: + goto s_n_llhttp__internal__n_req_after_protocol; + case 38: + goto s_n_llhttp__internal__n_req_after_protocol; + case 39: + goto s_n_llhttp__internal__n_req_after_protocol; + case 40: + goto s_n_llhttp__internal__n_req_after_protocol; + case 41: + goto s_n_llhttp__internal__n_req_after_protocol; + case 42: + goto s_n_llhttp__internal__n_req_after_protocol; + case 43: + goto s_n_llhttp__internal__n_req_after_protocol; + case 44: + goto s_n_llhttp__internal__n_req_after_protocol; + case 45: + goto s_n_llhttp__internal__n_req_after_protocol; + default: + goto s_n_llhttp__internal__n_error_81; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_2: + s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_2: { + switch (llhttp__on_protocol_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_load_method_3; + case 21: + goto s_n_llhttp__internal__n_pause_24; + default: + goto s_n_llhttp__internal__n_error_80; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_req_after_http_start_3: + s_n_llhttp__internal__n_req_after_http_start_3: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_req_after_http_start_3; } match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob16, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { p++; - goto s_n_llhttp__internal__n_invoke_load_method_2; + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_2; } case kMatchPause: { - return s_n_llhttp__internal__n_req_http_start_2; + return s_n_llhttp__internal__n_req_after_http_start_3; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_36; + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_3; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_req_http_start_3: - s_n_llhttp__internal__n_req_http_start_3: { - llparse_match_t match_seq; - + case s_n_llhttp__internal__n_req_after_http_start: + s_n_llhttp__internal__n_req_after_http_start: { if (p == endp) { - return s_n_llhttp__internal__n_req_http_start_3; + return s_n_llhttp__internal__n_req_after_http_start; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob17, 4); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { + switch (*p) { + case 'H': { p++; - goto s_n_llhttp__internal__n_invoke_load_method_3; + goto s_n_llhttp__internal__n_req_after_http_start_1; } - case kMatchPause: { - return s_n_llhttp__internal__n_req_http_start_3; + case 'I': { + p++; + goto s_n_llhttp__internal__n_req_after_http_start_2; } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_36; + case 'R': { + p++; + goto s_n_llhttp__internal__n_req_after_http_start_3; + } + default: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_3; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_span_start_llhttp__on_protocol: + s_n_llhttp__internal__n_span_start_llhttp__on_protocol: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_protocol; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_protocol; + goto s_n_llhttp__internal__n_req_after_http_start; + UNREACHABLE; } case s_n_llhttp__internal__n_req_http_start: s_n_llhttp__internal__n_req_http_start: { @@ -2875,24 +3846,11 @@ static llparse_state_t llhttp__internal__run( p++; goto s_n_llhttp__internal__n_req_http_start; } - case 'H': { - p++; - goto s_n_llhttp__internal__n_req_http_start_1; - } - case 'I': { - p++; - goto s_n_llhttp__internal__n_req_http_start_2; - } - case 'R': { - p++; - goto s_n_llhttp__internal__n_req_http_start_3; - } default: { - goto s_n_llhttp__internal__n_error_36; + goto s_n_llhttp__internal__n_span_start_llhttp__on_protocol; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_to_http: s_n_llhttp__internal__n_url_to_http: { @@ -2902,18 +3860,17 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 9: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 12: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } default: { goto s_n_llhttp__internal__n_invoke_llhttp__on_url_complete_1; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_skip_to_http: s_n_llhttp__internal__n_url_skip_to_http: { @@ -2923,19 +3880,18 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 9: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 12: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } default: { p++; goto s_n_llhttp__internal__n_url_to_http; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_fragment: s_n_llhttp__internal__n_url_fragment: { @@ -2963,7 +3919,7 @@ static llparse_state_t llhttp__internal__run( switch (lookup_table[(uint8_t) *p]) { case 1: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 2: { goto s_n_llhttp__internal__n_span_end_llhttp__on_url_6; @@ -2979,11 +3935,10 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_fragment; } default: { - goto s_n_llhttp__internal__n_error_37; + goto s_n_llhttp__internal__n_error_83; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_end_stub_query_3: s_n_llhttp__internal__n_span_end_stub_query_3: { @@ -2992,8 +3947,7 @@ static llparse_state_t llhttp__internal__run( } p++; goto s_n_llhttp__internal__n_url_fragment; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_query: s_n_llhttp__internal__n_url_query: { @@ -3021,7 +3975,7 @@ static llparse_state_t llhttp__internal__run( switch (lookup_table[(uint8_t) *p]) { case 1: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 2: { goto s_n_llhttp__internal__n_span_end_llhttp__on_url_9; @@ -3040,11 +3994,10 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_span_end_stub_query_3; } default: { - goto s_n_llhttp__internal__n_error_38; + goto s_n_llhttp__internal__n_error_84; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_query_or_fragment: s_n_llhttp__internal__n_url_query_or_fragment: { @@ -3054,14 +4007,14 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 9: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 10: { goto s_n_llhttp__internal__n_span_end_llhttp__on_url_3; } case 12: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 13: { goto s_n_llhttp__internal__n_span_end_llhttp__on_url_4; @@ -3078,11 +4031,10 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_query; } default: { - goto s_n_llhttp__internal__n_error_39; + goto s_n_llhttp__internal__n_error_85; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_path: s_n_llhttp__internal__n_url_path: { @@ -3110,7 +4062,7 @@ static llparse_state_t llhttp__internal__run( switch (lookup_table[(uint8_t) *p]) { case 1: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 2: { p++; @@ -3120,8 +4072,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_query_or_fragment; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_start_stub_path_2: s_n_llhttp__internal__n_span_start_stub_path_2: { @@ -3130,8 +4081,7 @@ static llparse_state_t llhttp__internal__run( } p++; goto s_n_llhttp__internal__n_url_path; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_start_stub_path: s_n_llhttp__internal__n_span_start_stub_path: { @@ -3140,8 +4090,7 @@ static llparse_state_t llhttp__internal__run( } p++; goto s_n_llhttp__internal__n_url_path; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_start_stub_path_1: s_n_llhttp__internal__n_span_start_stub_path_1: { @@ -3150,8 +4099,7 @@ static llparse_state_t llhttp__internal__run( } p++; goto s_n_llhttp__internal__n_url_path; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_server_with_at: s_n_llhttp__internal__n_url_server_with_at: { @@ -3179,7 +4127,7 @@ static llparse_state_t llhttp__internal__run( switch (lookup_table[(uint8_t) *p]) { case 1: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 2: { goto s_n_llhttp__internal__n_span_end_llhttp__on_url_12; @@ -3203,14 +4151,13 @@ static llparse_state_t llhttp__internal__run( } case 8: { p++; - goto s_n_llhttp__internal__n_error_40; + goto s_n_llhttp__internal__n_error_86; } default: { - goto s_n_llhttp__internal__n_error_41; + goto s_n_llhttp__internal__n_error_87; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_server: s_n_llhttp__internal__n_url_server: { @@ -3238,7 +4185,7 @@ static llparse_state_t llhttp__internal__run( switch (lookup_table[(uint8_t) *p]) { case 1: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 2: { goto s_n_llhttp__internal__n_span_end_llhttp__on_url; @@ -3265,11 +4212,10 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_server_with_at; } default: { - goto s_n_llhttp__internal__n_error_42; + goto s_n_llhttp__internal__n_error_88; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_schema_delim_1: s_n_llhttp__internal__n_url_schema_delim_1: { @@ -3282,11 +4228,10 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_server; } default: { - goto s_n_llhttp__internal__n_error_44; + goto s_n_llhttp__internal__n_error_89; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_schema_delim: s_n_llhttp__internal__n_url_schema_delim: { @@ -3296,34 +4241,33 @@ static llparse_state_t llhttp__internal__run( switch (*p) { case 9: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 10: { p++; - goto s_n_llhttp__internal__n_error_43; + goto s_n_llhttp__internal__n_error_2; } case 12: { p++; - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_2; } case 13: { p++; - goto s_n_llhttp__internal__n_error_43; + goto s_n_llhttp__internal__n_error_2; } case ' ': { p++; - goto s_n_llhttp__internal__n_error_43; + goto s_n_llhttp__internal__n_error_2; } case '/': { p++; goto s_n_llhttp__internal__n_url_schema_delim_1; } default: { - goto s_n_llhttp__internal__n_error_44; + goto s_n_llhttp__internal__n_error_89; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_end_stub_schema: s_n_llhttp__internal__n_span_end_stub_schema: { @@ -3332,7470 +4276,12 @@ static llparse_state_t llhttp__internal__run( } p++; goto s_n_llhttp__internal__n_url_schema_delim; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_schema: s_n_llhttp__internal__n_url_schema: { static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 2, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, - 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, - 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - if (p == endp) { - return s_n_llhttp__internal__n_url_schema; - } - switch (lookup_table[(uint8_t) *p]) { - case 1: { - p++; - goto s_n_llhttp__internal__n_error_1; - } - case 2: { - p++; - goto s_n_llhttp__internal__n_error_43; - } - case 3: { - goto s_n_llhttp__internal__n_span_end_stub_schema; - } - case 4: { - p++; - goto s_n_llhttp__internal__n_url_schema; - } - default: { - goto s_n_llhttp__internal__n_error_45; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_start: - s_n_llhttp__internal__n_url_start: { - static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 2, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, - 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - if (p == endp) { - return s_n_llhttp__internal__n_url_start; - } - switch (lookup_table[(uint8_t) *p]) { - case 1: { - p++; - goto s_n_llhttp__internal__n_error_1; - } - case 2: { - p++; - goto s_n_llhttp__internal__n_error_43; - } - case 3: { - goto s_n_llhttp__internal__n_span_start_stub_path_2; - } - case 4: { - goto s_n_llhttp__internal__n_url_schema; - } - default: { - goto s_n_llhttp__internal__n_error_46; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_start_llhttp__on_url_1: - s_n_llhttp__internal__n_span_start_llhttp__on_url_1: { - if (p == endp) { - return s_n_llhttp__internal__n_span_start_llhttp__on_url_1; - } - state->_span_pos0 = (void*) p; - state->_span_cb0 = llhttp__on_url; - goto s_n_llhttp__internal__n_url_start; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_entry_normal: - s_n_llhttp__internal__n_url_entry_normal: { - if (p == endp) { - return s_n_llhttp__internal__n_url_entry_normal; - } - switch (*p) { - case 9: { - p++; - goto s_n_llhttp__internal__n_error_1; - } - case 12: { - p++; - goto s_n_llhttp__internal__n_error_1; - } - default: { - goto s_n_llhttp__internal__n_span_start_llhttp__on_url_1; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_start_llhttp__on_url: - s_n_llhttp__internal__n_span_start_llhttp__on_url: { - if (p == endp) { - return s_n_llhttp__internal__n_span_start_llhttp__on_url; - } - state->_span_pos0 = (void*) p; - state->_span_cb0 = llhttp__on_url; - goto s_n_llhttp__internal__n_url_server; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_entry_connect: - s_n_llhttp__internal__n_url_entry_connect: { - if (p == endp) { - return s_n_llhttp__internal__n_url_entry_connect; - } - switch (*p) { - case 9: { - p++; - goto s_n_llhttp__internal__n_error_1; - } - case 12: { - p++; - goto s_n_llhttp__internal__n_error_1; - } - default: { - goto s_n_llhttp__internal__n_span_start_llhttp__on_url; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_spaces_before_url: - s_n_llhttp__internal__n_req_spaces_before_url: { - if (p == endp) { - return s_n_llhttp__internal__n_req_spaces_before_url; - } - switch (*p) { - case ' ': { - p++; - goto s_n_llhttp__internal__n_req_spaces_before_url; - } - default: { - goto s_n_llhttp__internal__n_invoke_is_equal_method; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_first_space_before_url: - s_n_llhttp__internal__n_req_first_space_before_url: { - if (p == endp) { - return s_n_llhttp__internal__n_req_first_space_before_url; - } - switch (*p) { - case ' ': { - p++; - goto s_n_llhttp__internal__n_req_spaces_before_url; - } - default: { - goto s_n_llhttp__internal__n_error_47; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_2: - s_n_llhttp__internal__n_start_req_2: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_2; - } - switch (*p) { - case 'L': { - p++; - match = 19; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_3: - s_n_llhttp__internal__n_start_req_3: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_3; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob18, 6); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 36; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_3; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_1: - s_n_llhttp__internal__n_start_req_1: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_1; - } - switch (*p) { - case 'C': { - p++; - goto s_n_llhttp__internal__n_start_req_2; - } - case 'N': { - p++; - goto s_n_llhttp__internal__n_start_req_3; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_4: - s_n_llhttp__internal__n_start_req_4: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_4; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob19, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 16; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_4; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_6: - s_n_llhttp__internal__n_start_req_6: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_6; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob20, 6); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 22; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_6; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_8: - s_n_llhttp__internal__n_start_req_8: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_8; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob21, 4); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 5; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_8; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_9: - s_n_llhttp__internal__n_start_req_9: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_9; - } - switch (*p) { - case 'Y': { - p++; - match = 8; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_7: - s_n_llhttp__internal__n_start_req_7: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_7; - } - switch (*p) { - case 'N': { - p++; - goto s_n_llhttp__internal__n_start_req_8; - } - case 'P': { - p++; - goto s_n_llhttp__internal__n_start_req_9; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_5: - s_n_llhttp__internal__n_start_req_5: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_5; - } - switch (*p) { - case 'H': { - p++; - goto s_n_llhttp__internal__n_start_req_6; - } - case 'O': { - p++; - goto s_n_llhttp__internal__n_start_req_7; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_12: - s_n_llhttp__internal__n_start_req_12: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_12; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob22, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 0; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_12; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_13: - s_n_llhttp__internal__n_start_req_13: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_13; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob23, 5); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 35; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_13; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_11: - s_n_llhttp__internal__n_start_req_11: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_11; - } - switch (*p) { - case 'L': { - p++; - goto s_n_llhttp__internal__n_start_req_12; - } - case 'S': { - p++; - goto s_n_llhttp__internal__n_start_req_13; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_10: - s_n_llhttp__internal__n_start_req_10: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_10; - } - switch (*p) { - case 'E': { - p++; - goto s_n_llhttp__internal__n_start_req_11; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_14: - s_n_llhttp__internal__n_start_req_14: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_14; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob24, 4); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 45; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_14; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_17: - s_n_llhttp__internal__n_start_req_17: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_17; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob26, 9); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 41; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_17; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_16: - s_n_llhttp__internal__n_start_req_16: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_16; - } - switch (*p) { - case '_': { - p++; - goto s_n_llhttp__internal__n_start_req_17; - } - default: { - match = 1; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_15: - s_n_llhttp__internal__n_start_req_15: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_15; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob25, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_start_req_16; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_15; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_18: - s_n_llhttp__internal__n_start_req_18: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_18; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob27, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 2; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_18; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_20: - s_n_llhttp__internal__n_start_req_20: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_20; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob28, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 31; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_20; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_21: - s_n_llhttp__internal__n_start_req_21: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_21; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob29, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 9; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_21; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_19: - s_n_llhttp__internal__n_start_req_19: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_19; - } - switch (*p) { - case 'I': { - p++; - goto s_n_llhttp__internal__n_start_req_20; - } - case 'O': { - p++; - goto s_n_llhttp__internal__n_start_req_21; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_23: - s_n_llhttp__internal__n_start_req_23: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_23; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob30, 6); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 24; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_23; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_24: - s_n_llhttp__internal__n_start_req_24: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_24; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob31, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 23; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_24; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_26: - s_n_llhttp__internal__n_start_req_26: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_26; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob32, 7); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 21; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_26; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_28: - s_n_llhttp__internal__n_start_req_28: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_28; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob33, 6); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 30; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_28; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_29: - s_n_llhttp__internal__n_start_req_29: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_29; - } - switch (*p) { - case 'L': { - p++; - match = 10; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_27: - s_n_llhttp__internal__n_start_req_27: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_27; - } - switch (*p) { - case 'A': { - p++; - goto s_n_llhttp__internal__n_start_req_28; - } - case 'O': { - p++; - goto s_n_llhttp__internal__n_start_req_29; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_25: - s_n_llhttp__internal__n_start_req_25: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_25; - } - switch (*p) { - case 'A': { - p++; - goto s_n_llhttp__internal__n_start_req_26; - } - case 'C': { - p++; - goto s_n_llhttp__internal__n_start_req_27; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_30: - s_n_llhttp__internal__n_start_req_30: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_30; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob34, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 11; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_30; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_22: - s_n_llhttp__internal__n_start_req_22: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_22; - } - switch (*p) { - case '-': { - p++; - goto s_n_llhttp__internal__n_start_req_23; - } - case 'E': { - p++; - goto s_n_llhttp__internal__n_start_req_24; - } - case 'K': { - p++; - goto s_n_llhttp__internal__n_start_req_25; - } - case 'O': { - p++; - goto s_n_llhttp__internal__n_start_req_30; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_31: - s_n_llhttp__internal__n_start_req_31: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_31; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob35, 5); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 25; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_31; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_32: - s_n_llhttp__internal__n_start_req_32: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_32; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob36, 6); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 6; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_32; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_35: - s_n_llhttp__internal__n_start_req_35: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_35; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob37, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 28; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_35; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_36: - s_n_llhttp__internal__n_start_req_36: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_36; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob38, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 39; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_36; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_34: - s_n_llhttp__internal__n_start_req_34: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_34; - } - switch (*p) { - case 'T': { - p++; - goto s_n_llhttp__internal__n_start_req_35; - } - case 'U': { - p++; - goto s_n_llhttp__internal__n_start_req_36; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_37: - s_n_llhttp__internal__n_start_req_37: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_37; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob39, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 38; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_37; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_38: - s_n_llhttp__internal__n_start_req_38: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_38; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob40, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 3; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_38; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_42: - s_n_llhttp__internal__n_start_req_42: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_42; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob41, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 12; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_42; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_43: - s_n_llhttp__internal__n_start_req_43: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_43; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob42, 4); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 13; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_43; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_41: - s_n_llhttp__internal__n_start_req_41: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_41; - } - switch (*p) { - case 'F': { - p++; - goto s_n_llhttp__internal__n_start_req_42; - } - case 'P': { - p++; - goto s_n_llhttp__internal__n_start_req_43; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_40: - s_n_llhttp__internal__n_start_req_40: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_40; - } - switch (*p) { - case 'P': { - p++; - goto s_n_llhttp__internal__n_start_req_41; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_39: - s_n_llhttp__internal__n_start_req_39: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_39; - } - switch (*p) { - case 'I': { - p++; - match = 34; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case 'O': { - p++; - goto s_n_llhttp__internal__n_start_req_40; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_45: - s_n_llhttp__internal__n_start_req_45: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_45; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob43, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 29; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_45; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_44: - s_n_llhttp__internal__n_start_req_44: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_44; - } - switch (*p) { - case 'R': { - p++; - goto s_n_llhttp__internal__n_start_req_45; - } - case 'T': { - p++; - match = 4; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_33: - s_n_llhttp__internal__n_start_req_33: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_33; - } - switch (*p) { - case 'A': { - p++; - goto s_n_llhttp__internal__n_start_req_34; - } - case 'L': { - p++; - goto s_n_llhttp__internal__n_start_req_37; - } - case 'O': { - p++; - goto s_n_llhttp__internal__n_start_req_38; - } - case 'R': { - p++; - goto s_n_llhttp__internal__n_start_req_39; - } - case 'U': { - p++; - goto s_n_llhttp__internal__n_start_req_44; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_48: - s_n_llhttp__internal__n_start_req_48: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_48; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob44, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 17; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_48; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_49: - s_n_llhttp__internal__n_start_req_49: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_49; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob45, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 44; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_49; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_50: - s_n_llhttp__internal__n_start_req_50: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_50; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob46, 5); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 43; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_50; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_51: - s_n_llhttp__internal__n_start_req_51: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_51; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob47, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 20; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_51; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_47: - s_n_llhttp__internal__n_start_req_47: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_47; - } - switch (*p) { - case 'B': { - p++; - goto s_n_llhttp__internal__n_start_req_48; - } - case 'C': { - p++; - goto s_n_llhttp__internal__n_start_req_49; - } - case 'D': { - p++; - goto s_n_llhttp__internal__n_start_req_50; - } - case 'P': { - p++; - goto s_n_llhttp__internal__n_start_req_51; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_46: - s_n_llhttp__internal__n_start_req_46: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_46; - } - switch (*p) { - case 'E': { - p++; - goto s_n_llhttp__internal__n_start_req_47; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_54: - s_n_llhttp__internal__n_start_req_54: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_54; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob48, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 14; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_54; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_56: - s_n_llhttp__internal__n_start_req_56: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_56; - } - switch (*p) { - case 'P': { - p++; - match = 37; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_57: - s_n_llhttp__internal__n_start_req_57: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_57; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob49, 9); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 42; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_57; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_55: - s_n_llhttp__internal__n_start_req_55: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_55; - } - switch (*p) { - case 'U': { - p++; - goto s_n_llhttp__internal__n_start_req_56; - } - case '_': { - p++; - goto s_n_llhttp__internal__n_start_req_57; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_53: - s_n_llhttp__internal__n_start_req_53: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_53; - } - switch (*p) { - case 'A': { - p++; - goto s_n_llhttp__internal__n_start_req_54; - } - case 'T': { - p++; - goto s_n_llhttp__internal__n_start_req_55; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_58: - s_n_llhttp__internal__n_start_req_58: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_58; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob50, 4); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 33; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_58; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_59: - s_n_llhttp__internal__n_start_req_59: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_59; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob51, 7); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 26; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_59; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_52: - s_n_llhttp__internal__n_start_req_52: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_52; - } - switch (*p) { - case 'E': { - p++; - goto s_n_llhttp__internal__n_start_req_53; - } - case 'O': { - p++; - goto s_n_llhttp__internal__n_start_req_58; - } - case 'U': { - p++; - goto s_n_llhttp__internal__n_start_req_59; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_61: - s_n_llhttp__internal__n_start_req_61: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_61; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob52, 6); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 40; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_61; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_62: - s_n_llhttp__internal__n_start_req_62: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_62; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob53, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 7; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_62; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_60: - s_n_llhttp__internal__n_start_req_60: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_60; - } - switch (*p) { - case 'E': { - p++; - goto s_n_llhttp__internal__n_start_req_61; - } - case 'R': { - p++; - goto s_n_llhttp__internal__n_start_req_62; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_65: - s_n_llhttp__internal__n_start_req_65: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_65; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob54, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 18; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_65; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_67: - s_n_llhttp__internal__n_start_req_67: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_67; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob55, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 32; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_67; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_68: - s_n_llhttp__internal__n_start_req_68: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_68; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob56, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 15; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_68; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_66: - s_n_llhttp__internal__n_start_req_66: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_66; - } - switch (*p) { - case 'I': { - p++; - goto s_n_llhttp__internal__n_start_req_67; - } - case 'O': { - p++; - goto s_n_llhttp__internal__n_start_req_68; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_69: - s_n_llhttp__internal__n_start_req_69: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_req_69; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob57, 8); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 27; - goto s_n_llhttp__internal__n_invoke_store_method_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_req_69; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_64: - s_n_llhttp__internal__n_start_req_64: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_64; - } - switch (*p) { - case 'B': { - p++; - goto s_n_llhttp__internal__n_start_req_65; - } - case 'L': { - p++; - goto s_n_llhttp__internal__n_start_req_66; - } - case 'S': { - p++; - goto s_n_llhttp__internal__n_start_req_69; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_63: - s_n_llhttp__internal__n_start_req_63: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_63; - } - switch (*p) { - case 'N': { - p++; - goto s_n_llhttp__internal__n_start_req_64; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req: - s_n_llhttp__internal__n_start_req: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req; - } - switch (*p) { - case 'A': { - p++; - goto s_n_llhttp__internal__n_start_req_1; - } - case 'B': { - p++; - goto s_n_llhttp__internal__n_start_req_4; - } - case 'C': { - p++; - goto s_n_llhttp__internal__n_start_req_5; - } - case 'D': { - p++; - goto s_n_llhttp__internal__n_start_req_10; - } - case 'F': { - p++; - goto s_n_llhttp__internal__n_start_req_14; - } - case 'G': { - p++; - goto s_n_llhttp__internal__n_start_req_15; - } - case 'H': { - p++; - goto s_n_llhttp__internal__n_start_req_18; - } - case 'L': { - p++; - goto s_n_llhttp__internal__n_start_req_19; - } - case 'M': { - p++; - goto s_n_llhttp__internal__n_start_req_22; - } - case 'N': { - p++; - goto s_n_llhttp__internal__n_start_req_31; - } - case 'O': { - p++; - goto s_n_llhttp__internal__n_start_req_32; - } - case 'P': { - p++; - goto s_n_llhttp__internal__n_start_req_33; - } - case 'R': { - p++; - goto s_n_llhttp__internal__n_start_req_46; - } - case 'S': { - p++; - goto s_n_llhttp__internal__n_start_req_52; - } - case 'T': { - p++; - goto s_n_llhttp__internal__n_start_req_60; - } - case 'U': { - p++; - goto s_n_llhttp__internal__n_start_req_63; - } - default: { - goto s_n_llhttp__internal__n_error_56; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_invoke_llhttp__on_status_complete: - s_n_llhttp__internal__n_invoke_llhttp__on_status_complete: { - switch (llhttp__on_status_complete(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_field_start; - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_res_line_almost_done: - s_n_llhttp__internal__n_res_line_almost_done: { - if (p == endp) { - return s_n_llhttp__internal__n_res_line_almost_done; - } - switch (*p) { - case 10: { - p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; - } - default: { - goto s_n_llhttp__internal__n_error_49; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_res_status: - s_n_llhttp__internal__n_res_status: { - if (p == endp) { - return s_n_llhttp__internal__n_res_status; - } - switch (*p) { - case 10: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_status; - } - case 13: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_status_1; - } - default: { - p++; - goto s_n_llhttp__internal__n_res_status; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_start_llhttp__on_status: - s_n_llhttp__internal__n_span_start_llhttp__on_status: { - if (p == endp) { - return s_n_llhttp__internal__n_span_start_llhttp__on_status; - } - state->_span_pos0 = (void*) p; - state->_span_cb0 = llhttp__on_status; - goto s_n_llhttp__internal__n_res_status; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_res_status_start: - s_n_llhttp__internal__n_res_status_start: { - if (p == endp) { - return s_n_llhttp__internal__n_res_status_start; - } - switch (*p) { - case 10: { - p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; - } - case 13: { - p++; - goto s_n_llhttp__internal__n_res_line_almost_done; - } - default: { - goto s_n_llhttp__internal__n_span_start_llhttp__on_status; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_res_status_code_otherwise: - s_n_llhttp__internal__n_res_status_code_otherwise: { - if (p == endp) { - return s_n_llhttp__internal__n_res_status_code_otherwise; - } - switch (*p) { - case 10: { - goto s_n_llhttp__internal__n_res_status_start; - } - case 13: { - goto s_n_llhttp__internal__n_res_status_start; - } - case ' ': { - p++; - goto s_n_llhttp__internal__n_res_status_start; - } - default: { - goto s_n_llhttp__internal__n_error_50; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_res_status_code: - s_n_llhttp__internal__n_res_status_code: { - if (p == endp) { - return s_n_llhttp__internal__n_res_status_code; - } - switch (*p) { - case '0': { - p++; - match = 0; - goto s_n_llhttp__internal__n_invoke_mul_add_status_code; - } - case '1': { - p++; - match = 1; - goto s_n_llhttp__internal__n_invoke_mul_add_status_code; - } - case '2': { - p++; - match = 2; - goto s_n_llhttp__internal__n_invoke_mul_add_status_code; - } - case '3': { - p++; - match = 3; - goto s_n_llhttp__internal__n_invoke_mul_add_status_code; - } - case '4': { - p++; - match = 4; - goto s_n_llhttp__internal__n_invoke_mul_add_status_code; - } - case '5': { - p++; - match = 5; - goto s_n_llhttp__internal__n_invoke_mul_add_status_code; - } - case '6': { - p++; - match = 6; - goto s_n_llhttp__internal__n_invoke_mul_add_status_code; - } - case '7': { - p++; - match = 7; - goto s_n_llhttp__internal__n_invoke_mul_add_status_code; - } - case '8': { - p++; - match = 8; - goto s_n_llhttp__internal__n_invoke_mul_add_status_code; - } - case '9': { - p++; - match = 9; - goto s_n_llhttp__internal__n_invoke_mul_add_status_code; - } - default: { - goto s_n_llhttp__internal__n_res_status_code_otherwise; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_res_http_end: - s_n_llhttp__internal__n_res_http_end: { - if (p == endp) { - return s_n_llhttp__internal__n_res_http_end; - } - switch (*p) { - case ' ': { - p++; - goto s_n_llhttp__internal__n_invoke_update_status_code; - } - default: { - goto s_n_llhttp__internal__n_error_51; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_res_http_minor: - s_n_llhttp__internal__n_res_http_minor: { - if (p == endp) { - return s_n_llhttp__internal__n_res_http_minor; - } - switch (*p) { - case '0': { - p++; - match = 0; - goto s_n_llhttp__internal__n_invoke_store_http_minor_1; - } - case '1': { - p++; - match = 1; - goto s_n_llhttp__internal__n_invoke_store_http_minor_1; - } - case '2': { - p++; - match = 2; - goto s_n_llhttp__internal__n_invoke_store_http_minor_1; - } - case '3': { - p++; - match = 3; - goto s_n_llhttp__internal__n_invoke_store_http_minor_1; - } - case '4': { - p++; - match = 4; - goto s_n_llhttp__internal__n_invoke_store_http_minor_1; - } - case '5': { - p++; - match = 5; - goto s_n_llhttp__internal__n_invoke_store_http_minor_1; - } - case '6': { - p++; - match = 6; - goto s_n_llhttp__internal__n_invoke_store_http_minor_1; - } - case '7': { - p++; - match = 7; - goto s_n_llhttp__internal__n_invoke_store_http_minor_1; - } - case '8': { - p++; - match = 8; - goto s_n_llhttp__internal__n_invoke_store_http_minor_1; - } - case '9': { - p++; - match = 9; - goto s_n_llhttp__internal__n_invoke_store_http_minor_1; - } - default: { - goto s_n_llhttp__internal__n_error_52; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_res_http_dot: - s_n_llhttp__internal__n_res_http_dot: { - if (p == endp) { - return s_n_llhttp__internal__n_res_http_dot; - } - switch (*p) { - case '.': { - p++; - goto s_n_llhttp__internal__n_res_http_minor; - } - default: { - goto s_n_llhttp__internal__n_error_53; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_res_http_major: - s_n_llhttp__internal__n_res_http_major: { - if (p == endp) { - return s_n_llhttp__internal__n_res_http_major; - } - switch (*p) { - case '0': { - p++; - match = 0; - goto s_n_llhttp__internal__n_invoke_store_http_major_1; - } - case '1': { - p++; - match = 1; - goto s_n_llhttp__internal__n_invoke_store_http_major_1; - } - case '2': { - p++; - match = 2; - goto s_n_llhttp__internal__n_invoke_store_http_major_1; - } - case '3': { - p++; - match = 3; - goto s_n_llhttp__internal__n_invoke_store_http_major_1; - } - case '4': { - p++; - match = 4; - goto s_n_llhttp__internal__n_invoke_store_http_major_1; - } - case '5': { - p++; - match = 5; - goto s_n_llhttp__internal__n_invoke_store_http_major_1; - } - case '6': { - p++; - match = 6; - goto s_n_llhttp__internal__n_invoke_store_http_major_1; - } - case '7': { - p++; - match = 7; - goto s_n_llhttp__internal__n_invoke_store_http_major_1; - } - case '8': { - p++; - match = 8; - goto s_n_llhttp__internal__n_invoke_store_http_major_1; - } - case '9': { - p++; - match = 9; - goto s_n_llhttp__internal__n_invoke_store_http_major_1; - } - default: { - goto s_n_llhttp__internal__n_error_54; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_res: - s_n_llhttp__internal__n_start_res: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_start_res; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob58, 5); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_res_http_major; - } - case kMatchPause: { - return s_n_llhttp__internal__n_start_res; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_57; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_or_res_method_2: - s_n_llhttp__internal__n_req_or_res_method_2: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_req_or_res_method_2; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob59, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 2; - goto s_n_llhttp__internal__n_invoke_store_method; - } - case kMatchPause: { - return s_n_llhttp__internal__n_req_or_res_method_2; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_55; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_or_res_method_3: - s_n_llhttp__internal__n_req_or_res_method_3: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_req_or_res_method_3; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob60, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_invoke_update_type_1; - } - case kMatchPause: { - return s_n_llhttp__internal__n_req_or_res_method_3; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_55; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_or_res_method_1: - s_n_llhttp__internal__n_req_or_res_method_1: { - if (p == endp) { - return s_n_llhttp__internal__n_req_or_res_method_1; - } - switch (*p) { - case 'E': { - p++; - goto s_n_llhttp__internal__n_req_or_res_method_2; - } - case 'T': { - p++; - goto s_n_llhttp__internal__n_req_or_res_method_3; - } - default: { - goto s_n_llhttp__internal__n_error_55; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_or_res_method: - s_n_llhttp__internal__n_req_or_res_method: { - if (p == endp) { - return s_n_llhttp__internal__n_req_or_res_method; - } - switch (*p) { - case 'H': { - p++; - goto s_n_llhttp__internal__n_req_or_res_method_1; - } - default: { - goto s_n_llhttp__internal__n_error_55; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start_req_or_res: - s_n_llhttp__internal__n_start_req_or_res: { - if (p == endp) { - return s_n_llhttp__internal__n_start_req_or_res; - } - switch (*p) { - case 'H': { - goto s_n_llhttp__internal__n_req_or_res_method; - } - default: { - goto s_n_llhttp__internal__n_invoke_update_type_2; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_invoke_load_type: - s_n_llhttp__internal__n_invoke_load_type: { - switch (llhttp__internal__c_load_type(state, p, endp)) { - case 1: - goto s_n_llhttp__internal__n_start_req; - case 2: - goto s_n_llhttp__internal__n_start_res; - default: - goto s_n_llhttp__internal__n_start_req_or_res; - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_start: - s_n_llhttp__internal__n_start: { - if (p == endp) { - return s_n_llhttp__internal__n_start; - } - switch (*p) { - case 10: { - p++; - goto s_n_llhttp__internal__n_start; - } - case 13: { - p++; - goto s_n_llhttp__internal__n_start; - } - default: { - goto s_n_llhttp__internal__n_invoke_update_finish; - } - } - /* UNREACHABLE */; - abort(); - } - default: - /* UNREACHABLE */ - abort(); - } - s_n_llhttp__internal__n_error_1: { - state->error = 0x7; - state->reason = "Invalid characters in url (strict mode)"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_43: { - state->error = 0x7; - state->reason = "Invalid characters in url"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_finish_2: { - switch (llhttp__internal__c_update_finish_1(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_start; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_4: { - state->error = 0x5; - state->reason = "Data after `Connection: close`"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_test_lenient_flags: { - switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { - case 1: - goto s_n_llhttp__internal__n_invoke_update_finish_2; - default: - goto s_n_llhttp__internal__n_closed; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_finish_1: { - switch (llhttp__internal__c_update_finish_1(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_test_lenient_flags; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_pause_5: { - state->error = 0x15; - state->reason = "on_message_complete pause"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_is_equal_upgrade; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_13: { - state->error = 0x12; - state->reason = "`on_message_complete` callback error"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_pause_7: { - state->error = 0x15; - state->reason = "on_chunk_complete pause"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_16: { - state->error = 0x14; - state->reason = "`on_chunk_complete` callback error"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_1: { - switch (llhttp__on_chunk_complete(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; - case 21: - goto s_n_llhttp__internal__n_pause_7; - default: - goto s_n_llhttp__internal__n_error_16; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_15: { - state->error = 0x4; - state->reason = "Content-Length can't be present with Transfer-Encoding"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_pause_2: { - state->error = 0x15; - state->reason = "on_message_complete pause"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_pause_1; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_5: { - state->error = 0x12; - state->reason = "`on_message_complete` callback error"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_1: { - switch (llhttp__on_message_complete(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_pause_1; - case 21: - goto s_n_llhttp__internal__n_pause_2; - default: - goto s_n_llhttp__internal__n_error_5; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_11: { - state->error = 0xc; - state->reason = "Chunk size overflow"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_pause_3: { - state->error = 0x15; - state->reason = "on_chunk_complete pause"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_update_content_length; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_7: { - state->error = 0x14; - state->reason = "`on_chunk_complete` callback error"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete: { - switch (llhttp__on_chunk_complete(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_invoke_update_content_length; - case 21: - goto s_n_llhttp__internal__n_pause_3; - default: - goto s_n_llhttp__internal__n_error_7; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_8: { - state->error = 0x2; - state->reason = "Expected CRLF after chunk"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_body: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_body(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_data_almost_done; - return s_error; - } - goto s_n_llhttp__internal__n_chunk_data_almost_done; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags: { - switch (llhttp__internal__c_or_flags(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_field_start; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_pause_4: { - state->error = 0x15; - state->reason = "on_chunk_header pause"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_is_equal_content_length; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_6: { - state->error = 0x13; - state->reason = "`on_chunk_header` callback error"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_llhttp__on_chunk_header: { - switch (llhttp__on_chunk_header(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_invoke_is_equal_content_length; - case 21: - goto s_n_llhttp__internal__n_pause_4; - default: - goto s_n_llhttp__internal__n_error_6; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_9: { - state->error = 0x2; - state->reason = "Expected LF after chunk size"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_10: { - state->error = 0xc; - state->reason = "Invalid character in chunk size"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_mul_add_content_length: { - switch (llhttp__internal__c_mul_add_content_length(state, p, endp, match)) { - case 1: - goto s_n_llhttp__internal__n_error_11; - default: - goto s_n_llhttp__internal__n_chunk_size; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_12: { - state->error = 0xc; - state->reason = "Invalid character in chunk size"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_body_1: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_body(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; - return s_error; - } - goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_finish_3: { - switch (llhttp__internal__c_update_finish_3(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_span_start_llhttp__on_body_2; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_14: { - state->error = 0xf; - state->reason = "Request has invalid `Transfer-Encoding`"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_pause: { - state->error = 0x15; - state->reason = "on_message_complete pause"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_3: { - state->error = 0x12; - state->reason = "`on_message_complete` callback error"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_llhttp__on_message_complete: { - switch (llhttp__on_message_complete(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; - case 21: - goto s_n_llhttp__internal__n_pause; - default: - goto s_n_llhttp__internal__n_error_3; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_1: { - switch (llhttp__internal__c_or_flags_1(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_2: { - switch (llhttp__internal__c_or_flags_1(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_upgrade: { - switch (llhttp__internal__c_update_upgrade(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_or_flags_2; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_pause_6: { - state->error = 0x15; - state->reason = "Paused by on_headers_complete"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_2: { - state->error = 0x11; - state->reason = "User callback error"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete: { - switch (llhttp__on_headers_complete(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; - case 1: - goto s_n_llhttp__internal__n_invoke_or_flags_1; - case 2: - goto s_n_llhttp__internal__n_invoke_update_upgrade; - case 21: - goto s_n_llhttp__internal__n_pause_6; - default: - goto s_n_llhttp__internal__n_error_2; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete: { - switch (llhttp__before_headers_complete(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_test_lenient_flags_1: { - switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_error_15; - default: - goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_test_flags_1: { - switch (llhttp__internal__c_test_flags_1(state, p, endp)) { - case 1: - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_1; - default: - goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_test_flags: { - switch (llhttp__internal__c_test_flags(state, p, endp)) { - case 1: - goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_1; - default: - goto s_n_llhttp__internal__n_invoke_test_flags_1; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_17: { - state->error = 0x2; - state->reason = "Expected LF after headers"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_18: { - state->error = 0xb; - state->reason = "Empty Content-Length"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_header_value(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; - return s_error; - } - goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_header_state: { - switch (llhttp__internal__c_update_header_state(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_3: { - switch (llhttp__internal__c_or_flags_3(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_4: { - switch (llhttp__internal__c_or_flags_4(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_5: { - switch (llhttp__internal__c_or_flags_5(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_6: { - switch (llhttp__internal__c_or_flags_6(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_load_header_state_1: { - switch (llhttp__internal__c_load_header_state(state, p, endp)) { - case 5: - goto s_n_llhttp__internal__n_invoke_or_flags_3; - case 6: - goto s_n_llhttp__internal__n_invoke_or_flags_4; - case 7: - goto s_n_llhttp__internal__n_invoke_or_flags_5; - case 8: - goto s_n_llhttp__internal__n_invoke_or_flags_6; - default: - goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_load_header_state: { - switch (llhttp__internal__c_load_header_state(state, p, endp)) { - case 2: - goto s_n_llhttp__internal__n_error_18; - default: - goto s_n_llhttp__internal__n_invoke_load_header_state_1; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_19: { - state->error = 0x2; - state->reason = "Expected LF after CR"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_header_state_1: { - switch (llhttp__internal__c_update_header_state(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_7: { - switch (llhttp__internal__c_or_flags_3(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state_1; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_8: { - switch (llhttp__internal__c_or_flags_4(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state_1; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_9: { - switch (llhttp__internal__c_or_flags_5(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state_1; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_10: { - switch (llhttp__internal__c_or_flags_6(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_load_header_state_3: { - switch (llhttp__internal__c_load_header_state(state, p, endp)) { - case 5: - goto s_n_llhttp__internal__n_invoke_or_flags_7; - case 6: - goto s_n_llhttp__internal__n_invoke_or_flags_8; - case 7: - goto s_n_llhttp__internal__n_invoke_or_flags_9; - case 8: - goto s_n_llhttp__internal__n_invoke_or_flags_10; - default: - goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_20: { - state->error = 0x3; - state->reason = "Missing expected LF after header value"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_header_value(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_almost_done; - return s_error; - } - goto s_n_llhttp__internal__n_header_value_almost_done; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_header_value(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_almost_done; - return s_error; - } - p++; - goto s_n_llhttp__internal__n_header_value_almost_done; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_header_value(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_almost_done; - return s_error; - } - p++; - goto s_n_llhttp__internal__n_header_value_almost_done; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_21: { - state->error = 0xa; - state->reason = "Invalid header value char"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_test_lenient_flags_2: { - switch (llhttp__internal__c_test_lenient_flags_2(state, p, endp)) { - case 1: - goto s_n_llhttp__internal__n_header_value_lenient; - default: - goto s_n_llhttp__internal__n_error_21; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_header_state_3: { - switch (llhttp__internal__c_update_header_state(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_value_connection; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_11: { - switch (llhttp__internal__c_or_flags_3(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state_3; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_12: { - switch (llhttp__internal__c_or_flags_4(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state_3; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_13: { - switch (llhttp__internal__c_or_flags_5(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state_3; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_14: { - switch (llhttp__internal__c_or_flags_6(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_value_connection; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_load_header_state_4: { - switch (llhttp__internal__c_load_header_state(state, p, endp)) { - case 5: - goto s_n_llhttp__internal__n_invoke_or_flags_11; - case 6: - goto s_n_llhttp__internal__n_invoke_or_flags_12; - case 7: - goto s_n_llhttp__internal__n_invoke_or_flags_13; - case 8: - goto s_n_llhttp__internal__n_invoke_or_flags_14; - default: - goto s_n_llhttp__internal__n_header_value_connection; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_header_state_4: { - switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_value_connection_token; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_header_state_2: { - switch (llhttp__internal__c_update_header_state_2(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_value_connection_ws; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_header_state_5: { - switch (llhttp__internal__c_update_header_state_5(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_value_connection_ws; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_header_state_6: { - switch (llhttp__internal__c_update_header_state_6(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_value_connection_ws; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_header_value(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_23; - return s_error; - } - goto s_n_llhttp__internal__n_error_23; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_mul_add_content_length_1: { - switch (llhttp__internal__c_mul_add_content_length_1(state, p, endp, match)) { - case 1: - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4; - default: - goto s_n_llhttp__internal__n_header_value_content_length; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_15: { - switch (llhttp__internal__c_or_flags_15(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_value_otherwise; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_header_value(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_24; - return s_error; - } - goto s_n_llhttp__internal__n_error_24; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_22: { - state->error = 0x4; - state->reason = "Duplicate Content-Length"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_test_flags_2: { - switch (llhttp__internal__c_test_flags_2(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_header_value_content_length; - default: - goto s_n_llhttp__internal__n_error_22; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_header_state_7: { - switch (llhttp__internal__c_update_header_state_7(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_value_otherwise; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_header_state_8: { - switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_value; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_and_flags: { - switch (llhttp__internal__c_and_flags(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_value_te_chunked; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_16: { - switch (llhttp__internal__c_or_flags_16(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_and_flags; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_17: { - switch (llhttp__internal__c_or_flags_17(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state_8; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_load_header_state_2: { - switch (llhttp__internal__c_load_header_state(state, p, endp)) { - case 1: - goto s_n_llhttp__internal__n_header_value_connection; - case 2: - goto s_n_llhttp__internal__n_invoke_test_flags_2; - case 3: - goto s_n_llhttp__internal__n_invoke_or_flags_16; - case 4: - goto s_n_llhttp__internal__n_invoke_or_flags_17; - default: - goto s_n_llhttp__internal__n_header_value; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_header_field: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_header_field(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; - return s_error; - } - p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_header_field_1: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_header_field(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; - return s_error; - } - p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_25: { - state->error = 0xa; - state->reason = "Invalid header token"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_header_state_9: { - switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_field_general; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_store_header_state: { - switch (llhttp__internal__c_store_header_state(state, p, endp, match)) { - default: - goto s_n_llhttp__internal__n_header_field_colon; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_header_state_10: { - switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_field_general; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_llhttp__on_url_complete: { - switch (llhttp__on_url_complete(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_field_start; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_http_minor: { - switch (llhttp__internal__c_update_http_minor(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_llhttp__on_url_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_http_major: { - switch (llhttp__internal__c_update_http_major(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_http_minor; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_3: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_to_http09; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_26: { - state->error = 0x7; - state->reason = "Expected CRLF"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_4: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_lf_to_http09; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_29: { - state->error = 0x17; - state->reason = "Pause on PRI/Upgrade"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_30: { - state->error = 0x9; - state->reason = "Expected HTTP/2 Connection Preface"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_28: { - state->error = 0x9; - state->reason = "Expected CRLF after version"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_load_method_1: { - switch (llhttp__internal__c_load_method(state, p, endp)) { - case 34: - goto s_n_llhttp__internal__n_req_pri_upgrade; - default: - goto s_n_llhttp__internal__n_req_http_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_store_http_minor: { - switch (llhttp__internal__c_store_http_minor(state, p, endp, match)) { - default: - goto s_n_llhttp__internal__n_invoke_load_method_1; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_31: { - state->error = 0x9; - state->reason = "Invalid minor version"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_32: { - state->error = 0x9; - state->reason = "Expected dot"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_store_http_major: { - switch (llhttp__internal__c_store_http_major(state, p, endp, match)) { - default: - goto s_n_llhttp__internal__n_req_http_dot; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_33: { - state->error = 0x9; - state->reason = "Invalid major version"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_27: { - state->error = 0x8; - state->reason = "Invalid method for HTTP/x.x request"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_load_method: { - switch (llhttp__internal__c_load_method(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_req_http_major; - case 1: - goto s_n_llhttp__internal__n_req_http_major; - case 2: - goto s_n_llhttp__internal__n_req_http_major; - case 3: - goto s_n_llhttp__internal__n_req_http_major; - case 4: - goto s_n_llhttp__internal__n_req_http_major; - case 5: - goto s_n_llhttp__internal__n_req_http_major; - case 6: - goto s_n_llhttp__internal__n_req_http_major; - case 7: - goto s_n_llhttp__internal__n_req_http_major; - case 8: - goto s_n_llhttp__internal__n_req_http_major; - case 9: - goto s_n_llhttp__internal__n_req_http_major; - case 10: - goto s_n_llhttp__internal__n_req_http_major; - case 11: - goto s_n_llhttp__internal__n_req_http_major; - case 12: - goto s_n_llhttp__internal__n_req_http_major; - case 13: - goto s_n_llhttp__internal__n_req_http_major; - case 14: - goto s_n_llhttp__internal__n_req_http_major; - case 15: - goto s_n_llhttp__internal__n_req_http_major; - case 16: - goto s_n_llhttp__internal__n_req_http_major; - case 17: - goto s_n_llhttp__internal__n_req_http_major; - case 18: - goto s_n_llhttp__internal__n_req_http_major; - case 19: - goto s_n_llhttp__internal__n_req_http_major; - case 20: - goto s_n_llhttp__internal__n_req_http_major; - case 21: - goto s_n_llhttp__internal__n_req_http_major; - case 22: - goto s_n_llhttp__internal__n_req_http_major; - case 23: - goto s_n_llhttp__internal__n_req_http_major; - case 24: - goto s_n_llhttp__internal__n_req_http_major; - case 25: - goto s_n_llhttp__internal__n_req_http_major; - case 26: - goto s_n_llhttp__internal__n_req_http_major; - case 27: - goto s_n_llhttp__internal__n_req_http_major; - case 28: - goto s_n_llhttp__internal__n_req_http_major; - case 29: - goto s_n_llhttp__internal__n_req_http_major; - case 30: - goto s_n_llhttp__internal__n_req_http_major; - case 31: - goto s_n_llhttp__internal__n_req_http_major; - case 32: - goto s_n_llhttp__internal__n_req_http_major; - case 33: - goto s_n_llhttp__internal__n_req_http_major; - case 34: - goto s_n_llhttp__internal__n_req_http_major; - default: - goto s_n_llhttp__internal__n_error_27; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_36: { - state->error = 0x8; - state->reason = "Expected HTTP/"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_34: { - state->error = 0x8; - state->reason = "Expected SOURCE method for ICE/x.x request"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_load_method_2: { - switch (llhttp__internal__c_load_method(state, p, endp)) { - case 33: - goto s_n_llhttp__internal__n_req_http_major; - default: - goto s_n_llhttp__internal__n_error_34; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_35: { - state->error = 0x8; - state->reason = "Invalid method for RTSP/x.x request"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_load_method_3: { - switch (llhttp__internal__c_load_method(state, p, endp)) { - case 1: - goto s_n_llhttp__internal__n_req_http_major; - case 3: - goto s_n_llhttp__internal__n_req_http_major; - case 6: - goto s_n_llhttp__internal__n_req_http_major; - case 35: - goto s_n_llhttp__internal__n_req_http_major; - case 36: - goto s_n_llhttp__internal__n_req_http_major; - case 37: - goto s_n_llhttp__internal__n_req_http_major; - case 38: - goto s_n_llhttp__internal__n_req_http_major; - case 39: - goto s_n_llhttp__internal__n_req_http_major; - case 40: - goto s_n_llhttp__internal__n_req_http_major; - case 41: - goto s_n_llhttp__internal__n_req_http_major; - case 42: - goto s_n_llhttp__internal__n_req_http_major; - case 43: - goto s_n_llhttp__internal__n_req_http_major; - case 44: - goto s_n_llhttp__internal__n_req_http_major; - case 45: - goto s_n_llhttp__internal__n_req_http_major; - default: - goto s_n_llhttp__internal__n_error_35; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_llhttp__on_url_complete_1: { - switch (llhttp__on_url_complete(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_req_http_start; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_5: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_to_http; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_6: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_to_http09; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_7: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_lf_to_http09; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_8: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_to_http; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_37: { - state->error = 0x7; - state->reason = "Invalid char in url fragment start"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_9: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_to_http09; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_10: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_lf_to_http09; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_11: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_to_http; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_38: { - state->error = 0x7; - state->reason = "Invalid char in url query"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_39: { - state->error = 0x7; - state->reason = "Invalid char in url path"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_to_http09; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_1: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_lf_to_http09; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_2: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_to_http; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_12: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http09; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_to_http09; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_13: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_lf_to_http09; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_lf_to_http09; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_url_14: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_url(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_url_skip_to_http; - return s_error; - } - goto s_n_llhttp__internal__n_url_skip_to_http; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_40: { - state->error = 0x7; - state->reason = "Double @ in url"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_41: { - state->error = 0x7; - state->reason = "Unexpected char in url server"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_42: { - state->error = 0x7; - state->reason = "Unexpected char in url server"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_44: { - state->error = 0x7; - state->reason = "Unexpected char in url schema"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_45: { - state->error = 0x7; - state->reason = "Unexpected char in url schema"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_46: { - state->error = 0x7; - state->reason = "Unexpected start char in url"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_is_equal_method: { - switch (llhttp__internal__c_is_equal_method(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_url_entry_normal; - default: - goto s_n_llhttp__internal__n_url_entry_connect; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_47: { - state->error = 0x6; - state->reason = "Expected space after method"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_store_method_1: { - switch (llhttp__internal__c_store_method(state, p, endp, match)) { - default: - goto s_n_llhttp__internal__n_req_first_space_before_url; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_56: { - state->error = 0x6; - state->reason = "Invalid method encountered"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_48: { - state->error = 0xd; - state->reason = "Response overflow"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_mul_add_status_code: { - switch (llhttp__internal__c_mul_add_status_code(state, p, endp, match)) { - case 1: - goto s_n_llhttp__internal__n_error_48; - default: - goto s_n_llhttp__internal__n_res_status_code; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_49: { - state->error = 0x2; - state->reason = "Expected LF after CR"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_status: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_status(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; - return s_error; - } - p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_span_end_llhttp__on_status_1: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_status(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_res_line_almost_done; - return s_error; - } - p++; - goto s_n_llhttp__internal__n_res_line_almost_done; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_50: { - state->error = 0xd; - state->reason = "Invalid response status"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_status_code: { - switch (llhttp__internal__c_update_status_code(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_res_status_code; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_51: { - state->error = 0x9; - state->reason = "Expected space after version"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_store_http_minor_1: { - switch (llhttp__internal__c_store_http_minor(state, p, endp, match)) { - default: - goto s_n_llhttp__internal__n_res_http_end; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_52: { - state->error = 0x9; - state->reason = "Invalid minor version"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_53: { - state->error = 0x9; - state->reason = "Expected dot"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_store_http_major_1: { - switch (llhttp__internal__c_store_http_major(state, p, endp, match)) { - default: - goto s_n_llhttp__internal__n_res_http_dot; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_54: { - state->error = 0x9; - state->reason = "Invalid major version"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_57: { - state->error = 0x8; - state->reason = "Expected HTTP/"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_type: { - switch (llhttp__internal__c_update_type(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_req_first_space_before_url; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_store_method: { - switch (llhttp__internal__c_store_method(state, p, endp, match)) { - default: - goto s_n_llhttp__internal__n_invoke_update_type; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_55: { - state->error = 0x8; - state->reason = "Invalid word encountered"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_type_1: { - switch (llhttp__internal__c_update_type_1(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_res_http_major; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_type_2: { - switch (llhttp__internal__c_update_type(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_start_req; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_pause_8: { - state->error = 0x15; - state->reason = "on_message_begin pause"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_load_type; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error: { - state->error = 0x10; - state->reason = "`on_message_begin` callback error"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_llhttp__on_message_begin: { - switch (llhttp__on_message_begin(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_invoke_load_type; - case 21: - goto s_n_llhttp__internal__n_pause_8; - default: - goto s_n_llhttp__internal__n_error; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_finish: { - switch (llhttp__internal__c_update_finish(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_llhttp__on_message_begin; - } - /* UNREACHABLE */; - abort(); - } -} - -int llhttp__internal_execute(llhttp__internal_t* state, const char* p, const char* endp) { - llparse_state_t next; - - /* check lingering errors */ - if (state->error != 0) { - return state->error; - } - - /* restart spans */ - if (state->_span_pos0 != NULL) { - state->_span_pos0 = (void*) p; - } - - next = llhttp__internal__run(state, (const unsigned char*) p, (const unsigned char*) endp); - if (next == s_error) { - return state->error; - } - state->_current = (void*) (intptr_t) next; - - /* execute spans */ - if (state->_span_pos0 != NULL) { - int error; - - error = ((llhttp__internal__span_cb) state->_span_cb0)(state, state->_span_pos0, (const char*) endp); - if (error != 0) { - state->error = error; - state->error_pos = endp; - return error; - } - } - - return 0; -} - -#else /* !LLHTTP_STRICT_MODE */ - -#include -#include -#include - -#ifdef __SSE4_2__ - #ifdef _MSC_VER - #include - #else /* !_MSC_VER */ - #include - #endif /* _MSC_VER */ -#endif /* __SSE4_2__ */ - -#ifdef _MSC_VER - #define ALIGN(n) _declspec(align(n)) -#else /* !_MSC_VER */ - #define ALIGN(n) __attribute__((aligned(n))) -#endif /* _MSC_VER */ - -#include "llhttp.h" - -typedef int (*llhttp__internal__span_cb)( - llhttp__internal_t*, const char*, const char*); - -#ifdef __SSE4_2__ -static const unsigned char ALIGN(16) llparse_blob0[] = { - 0x9, 0x9, 0xc, 0xc, '!', '"', '$', '>', '@', '~', 0x80, - 0xff, 0x0, 0x0, 0x0, 0x0 -}; -#endif /* __SSE4_2__ */ -static const unsigned char llparse_blob1[] = { - 'o', 'n' -}; -static const unsigned char llparse_blob2[] = { - 'e', 'c', 't', 'i', 'o', 'n' -}; -static const unsigned char llparse_blob3[] = { - 'l', 'o', 's', 'e' -}; -static const unsigned char llparse_blob4[] = { - 'e', 'e', 'p', '-', 'a', 'l', 'i', 'v', 'e' -}; -static const unsigned char llparse_blob5[] = { - 'p', 'g', 'r', 'a', 'd', 'e' -}; -static const unsigned char llparse_blob6[] = { - 'c', 'h', 'u', 'n', 'k', 'e', 'd' -}; -#ifdef __SSE4_2__ -static const unsigned char ALIGN(16) llparse_blob7[] = { - 0x9, 0x9, ' ', '~', 0x80, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0 -}; -#endif /* __SSE4_2__ */ -#ifdef __SSE4_2__ -static const unsigned char ALIGN(16) llparse_blob8[] = { - ' ', '!', '#', '\'', '*', '+', '-', '.', '0', '9', 'A', - 'Z', '^', 'z', '|', '|' -}; -#endif /* __SSE4_2__ */ -#ifdef __SSE4_2__ -static const unsigned char ALIGN(16) llparse_blob9[] = { - '~', '~', 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0 -}; -#endif /* __SSE4_2__ */ -static const unsigned char llparse_blob10[] = { - 'e', 'n', 't', '-', 'l', 'e', 'n', 'g', 't', 'h' -}; -static const unsigned char llparse_blob11[] = { - 'r', 'o', 'x', 'y', '-', 'c', 'o', 'n', 'n', 'e', 'c', - 't', 'i', 'o', 'n' -}; -static const unsigned char llparse_blob12[] = { - 'r', 'a', 'n', 's', 'f', 'e', 'r', '-', 'e', 'n', 'c', - 'o', 'd', 'i', 'n', 'g' -}; -static const unsigned char llparse_blob13[] = { - 'p', 'g', 'r', 'a', 'd', 'e' -}; -static const unsigned char llparse_blob14[] = { - 0xd, 0xa -}; -static const unsigned char llparse_blob15[] = { - 'T', 'T', 'P', '/' -}; -static const unsigned char llparse_blob16[] = { - 0xd, 0xa, 0xd, 0xa, 'S', 'M', 0xd, 0xa, 0xd, 0xa -}; -static const unsigned char llparse_blob17[] = { - 'C', 'E', '/' -}; -static const unsigned char llparse_blob18[] = { - 'T', 'S', 'P', '/' -}; -static const unsigned char llparse_blob19[] = { - 'N', 'O', 'U', 'N', 'C', 'E' -}; -static const unsigned char llparse_blob20[] = { - 'I', 'N', 'D' -}; -static const unsigned char llparse_blob21[] = { - 'E', 'C', 'K', 'O', 'U', 'T' -}; -static const unsigned char llparse_blob22[] = { - 'N', 'E', 'C', 'T' -}; -static const unsigned char llparse_blob23[] = { - 'E', 'T', 'E' -}; -static const unsigned char llparse_blob24[] = { - 'C', 'R', 'I', 'B', 'E' -}; -static const unsigned char llparse_blob25[] = { - 'L', 'U', 'S', 'H' -}; -static const unsigned char llparse_blob26[] = { - 'E', 'T' -}; -static const unsigned char llparse_blob27[] = { - 'P', 'A', 'R', 'A', 'M', 'E', 'T', 'E', 'R' -}; -static const unsigned char llparse_blob28[] = { - 'E', 'A', 'D' -}; -static const unsigned char llparse_blob29[] = { - 'N', 'K' -}; -static const unsigned char llparse_blob30[] = { - 'C', 'K' -}; -static const unsigned char llparse_blob31[] = { - 'S', 'E', 'A', 'R', 'C', 'H' -}; -static const unsigned char llparse_blob32[] = { - 'R', 'G', 'E' -}; -static const unsigned char llparse_blob33[] = { - 'C', 'T', 'I', 'V', 'I', 'T', 'Y' -}; -static const unsigned char llparse_blob34[] = { - 'L', 'E', 'N', 'D', 'A', 'R' -}; -static const unsigned char llparse_blob35[] = { - 'V', 'E' -}; -static const unsigned char llparse_blob36[] = { - 'O', 'T', 'I', 'F', 'Y' -}; -static const unsigned char llparse_blob37[] = { - 'P', 'T', 'I', 'O', 'N', 'S' -}; -static const unsigned char llparse_blob38[] = { - 'C', 'H' -}; -static const unsigned char llparse_blob39[] = { - 'S', 'E' -}; -static const unsigned char llparse_blob40[] = { - 'A', 'Y' -}; -static const unsigned char llparse_blob41[] = { - 'S', 'T' -}; -static const unsigned char llparse_blob42[] = { - 'I', 'N', 'D' -}; -static const unsigned char llparse_blob43[] = { - 'A', 'T', 'C', 'H' -}; -static const unsigned char llparse_blob44[] = { - 'G', 'E' -}; -static const unsigned char llparse_blob45[] = { - 'I', 'N', 'D' -}; -static const unsigned char llparse_blob46[] = { - 'O', 'R', 'D' -}; -static const unsigned char llparse_blob47[] = { - 'I', 'R', 'E', 'C', 'T' -}; -static const unsigned char llparse_blob48[] = { - 'O', 'R', 'T' -}; -static const unsigned char llparse_blob49[] = { - 'R', 'C', 'H' -}; -static const unsigned char llparse_blob50[] = { - 'P', 'A', 'R', 'A', 'M', 'E', 'T', 'E', 'R' -}; -static const unsigned char llparse_blob51[] = { - 'U', 'R', 'C', 'E' -}; -static const unsigned char llparse_blob52[] = { - 'B', 'S', 'C', 'R', 'I', 'B', 'E' -}; -static const unsigned char llparse_blob53[] = { - 'A', 'R', 'D', 'O', 'W', 'N' -}; -static const unsigned char llparse_blob54[] = { - 'A', 'C', 'E' -}; -static const unsigned char llparse_blob55[] = { - 'I', 'N', 'D' -}; -static const unsigned char llparse_blob56[] = { - 'N', 'K' -}; -static const unsigned char llparse_blob57[] = { - 'C', 'K' -}; -static const unsigned char llparse_blob58[] = { - 'U', 'B', 'S', 'C', 'R', 'I', 'B', 'E' -}; -static const unsigned char llparse_blob59[] = { - 'H', 'T', 'T', 'P', '/' -}; -static const unsigned char llparse_blob60[] = { - 'A', 'D' -}; -static const unsigned char llparse_blob61[] = { - 'T', 'P', '/' -}; - -enum llparse_match_status_e { - kMatchComplete, - kMatchPause, - kMatchMismatch -}; -typedef enum llparse_match_status_e llparse_match_status_t; - -struct llparse_match_s { - llparse_match_status_t status; - const unsigned char* current; -}; -typedef struct llparse_match_s llparse_match_t; - -static llparse_match_t llparse__match_sequence_to_lower( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp, - const unsigned char* seq, uint32_t seq_len) { - uint32_t index; - llparse_match_t res; - - index = s->_index; - for (; p != endp; p++) { - unsigned char current; - - current = ((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p)); - if (current == seq[index]) { - if (++index == seq_len) { - res.status = kMatchComplete; - goto reset; - } - } else { - res.status = kMatchMismatch; - goto reset; - } - } - s->_index = index; - res.status = kMatchPause; - res.current = p; - return res; -reset: - s->_index = 0; - res.current = p; - return res; -} - -static llparse_match_t llparse__match_sequence_to_lower_unsafe( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp, - const unsigned char* seq, uint32_t seq_len) { - uint32_t index; - llparse_match_t res; - - index = s->_index; - for (; p != endp; p++) { - unsigned char current; - - current = ((*p) | 0x20); - if (current == seq[index]) { - if (++index == seq_len) { - res.status = kMatchComplete; - goto reset; - } - } else { - res.status = kMatchMismatch; - goto reset; - } - } - s->_index = index; - res.status = kMatchPause; - res.current = p; - return res; -reset: - s->_index = 0; - res.current = p; - return res; -} - -static llparse_match_t llparse__match_sequence_id( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp, - const unsigned char* seq, uint32_t seq_len) { - uint32_t index; - llparse_match_t res; - - index = s->_index; - for (; p != endp; p++) { - unsigned char current; - - current = *p; - if (current == seq[index]) { - if (++index == seq_len) { - res.status = kMatchComplete; - goto reset; - } - } else { - res.status = kMatchMismatch; - goto reset; - } - } - s->_index = index; - res.status = kMatchPause; - res.current = p; - return res; -reset: - s->_index = 0; - res.current = p; - return res; -} - -enum llparse_state_e { - s_error, - s_n_llhttp__internal__n_closed, - s_n_llhttp__internal__n_invoke_llhttp__after_message_complete, - s_n_llhttp__internal__n_pause_1, - s_n_llhttp__internal__n_invoke_is_equal_upgrade, - s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2, - s_n_llhttp__internal__n_chunk_data_almost_done_skip, - s_n_llhttp__internal__n_chunk_data_almost_done, - s_n_llhttp__internal__n_consume_content_length, - s_n_llhttp__internal__n_span_start_llhttp__on_body, - s_n_llhttp__internal__n_invoke_is_equal_content_length, - s_n_llhttp__internal__n_chunk_size_almost_done, - s_n_llhttp__internal__n_chunk_parameters, - s_n_llhttp__internal__n_chunk_size_otherwise, - s_n_llhttp__internal__n_chunk_size, - s_n_llhttp__internal__n_chunk_size_digit, - s_n_llhttp__internal__n_invoke_update_content_length, - s_n_llhttp__internal__n_consume_content_length_1, - s_n_llhttp__internal__n_span_start_llhttp__on_body_1, - s_n_llhttp__internal__n_eof, - s_n_llhttp__internal__n_span_start_llhttp__on_body_2, - s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete, - s_n_llhttp__internal__n_headers_almost_done, - s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete, - s_n_llhttp__internal__n_span_start_llhttp__on_header_value, - s_n_llhttp__internal__n_header_value_discard_lws, - s_n_llhttp__internal__n_header_value_discard_ws_almost_done, - s_n_llhttp__internal__n_header_value_lws, - s_n_llhttp__internal__n_header_value_almost_done, - s_n_llhttp__internal__n_header_value_lenient, - s_n_llhttp__internal__n_header_value_otherwise, - s_n_llhttp__internal__n_header_value_connection_token, - s_n_llhttp__internal__n_header_value_connection_ws, - s_n_llhttp__internal__n_header_value_connection_1, - s_n_llhttp__internal__n_header_value_connection_2, - s_n_llhttp__internal__n_header_value_connection_3, - s_n_llhttp__internal__n_header_value_connection, - s_n_llhttp__internal__n_error_17, - s_n_llhttp__internal__n_error_18, - s_n_llhttp__internal__n_header_value_content_length_ws, - s_n_llhttp__internal__n_header_value_content_length, - s_n_llhttp__internal__n_header_value_te_chunked_last, - s_n_llhttp__internal__n_header_value_te_token_ows, - s_n_llhttp__internal__n_header_value, - s_n_llhttp__internal__n_header_value_te_token, - s_n_llhttp__internal__n_header_value_te_chunked, - s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1, - s_n_llhttp__internal__n_header_value_discard_ws, - s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete, - s_n_llhttp__internal__n_header_field_general_otherwise, - s_n_llhttp__internal__n_header_field_general, - s_n_llhttp__internal__n_header_field_colon, - s_n_llhttp__internal__n_header_field_3, - s_n_llhttp__internal__n_header_field_4, - s_n_llhttp__internal__n_header_field_2, - s_n_llhttp__internal__n_header_field_1, - s_n_llhttp__internal__n_header_field_5, - s_n_llhttp__internal__n_header_field_6, - s_n_llhttp__internal__n_header_field_7, - s_n_llhttp__internal__n_header_field, - s_n_llhttp__internal__n_span_start_llhttp__on_header_field, - s_n_llhttp__internal__n_header_field_start, - s_n_llhttp__internal__n_url_skip_to_http09, - s_n_llhttp__internal__n_url_skip_lf_to_http09, - s_n_llhttp__internal__n_req_pri_upgrade, - s_n_llhttp__internal__n_req_http_complete_1, - s_n_llhttp__internal__n_req_http_complete, - s_n_llhttp__internal__n_req_http_minor, - s_n_llhttp__internal__n_req_http_dot, - s_n_llhttp__internal__n_req_http_major, - s_n_llhttp__internal__n_req_http_start_1, - s_n_llhttp__internal__n_req_http_start_2, - s_n_llhttp__internal__n_req_http_start_3, - s_n_llhttp__internal__n_req_http_start, - s_n_llhttp__internal__n_url_skip_to_http, - s_n_llhttp__internal__n_url_fragment, - s_n_llhttp__internal__n_span_end_stub_query_3, - s_n_llhttp__internal__n_url_query, - s_n_llhttp__internal__n_url_query_or_fragment, - s_n_llhttp__internal__n_url_path, - s_n_llhttp__internal__n_span_start_stub_path_2, - s_n_llhttp__internal__n_span_start_stub_path, - s_n_llhttp__internal__n_span_start_stub_path_1, - s_n_llhttp__internal__n_url_server_with_at, - s_n_llhttp__internal__n_url_server, - s_n_llhttp__internal__n_url_schema_delim_1, - s_n_llhttp__internal__n_url_schema_delim, - s_n_llhttp__internal__n_span_end_stub_schema, - s_n_llhttp__internal__n_url_schema, - s_n_llhttp__internal__n_url_start, - s_n_llhttp__internal__n_span_start_llhttp__on_url_1, - s_n_llhttp__internal__n_span_start_llhttp__on_url, - s_n_llhttp__internal__n_req_spaces_before_url, - s_n_llhttp__internal__n_req_first_space_before_url, - s_n_llhttp__internal__n_start_req_2, - s_n_llhttp__internal__n_start_req_3, - s_n_llhttp__internal__n_start_req_1, - s_n_llhttp__internal__n_start_req_4, - s_n_llhttp__internal__n_start_req_6, - s_n_llhttp__internal__n_start_req_8, - s_n_llhttp__internal__n_start_req_9, - s_n_llhttp__internal__n_start_req_7, - s_n_llhttp__internal__n_start_req_5, - s_n_llhttp__internal__n_start_req_12, - s_n_llhttp__internal__n_start_req_13, - s_n_llhttp__internal__n_start_req_11, - s_n_llhttp__internal__n_start_req_10, - s_n_llhttp__internal__n_start_req_14, - s_n_llhttp__internal__n_start_req_17, - s_n_llhttp__internal__n_start_req_16, - s_n_llhttp__internal__n_start_req_15, - s_n_llhttp__internal__n_start_req_18, - s_n_llhttp__internal__n_start_req_20, - s_n_llhttp__internal__n_start_req_21, - s_n_llhttp__internal__n_start_req_19, - s_n_llhttp__internal__n_start_req_23, - s_n_llhttp__internal__n_start_req_24, - s_n_llhttp__internal__n_start_req_26, - s_n_llhttp__internal__n_start_req_28, - s_n_llhttp__internal__n_start_req_29, - s_n_llhttp__internal__n_start_req_27, - s_n_llhttp__internal__n_start_req_25, - s_n_llhttp__internal__n_start_req_30, - s_n_llhttp__internal__n_start_req_22, - s_n_llhttp__internal__n_start_req_31, - s_n_llhttp__internal__n_start_req_32, - s_n_llhttp__internal__n_start_req_35, - s_n_llhttp__internal__n_start_req_36, - s_n_llhttp__internal__n_start_req_34, - s_n_llhttp__internal__n_start_req_37, - s_n_llhttp__internal__n_start_req_38, - s_n_llhttp__internal__n_start_req_42, - s_n_llhttp__internal__n_start_req_43, - s_n_llhttp__internal__n_start_req_41, - s_n_llhttp__internal__n_start_req_40, - s_n_llhttp__internal__n_start_req_39, - s_n_llhttp__internal__n_start_req_45, - s_n_llhttp__internal__n_start_req_44, - s_n_llhttp__internal__n_start_req_33, - s_n_llhttp__internal__n_start_req_48, - s_n_llhttp__internal__n_start_req_49, - s_n_llhttp__internal__n_start_req_50, - s_n_llhttp__internal__n_start_req_51, - s_n_llhttp__internal__n_start_req_47, - s_n_llhttp__internal__n_start_req_46, - s_n_llhttp__internal__n_start_req_54, - s_n_llhttp__internal__n_start_req_56, - s_n_llhttp__internal__n_start_req_57, - s_n_llhttp__internal__n_start_req_55, - s_n_llhttp__internal__n_start_req_53, - s_n_llhttp__internal__n_start_req_58, - s_n_llhttp__internal__n_start_req_59, - s_n_llhttp__internal__n_start_req_52, - s_n_llhttp__internal__n_start_req_61, - s_n_llhttp__internal__n_start_req_62, - s_n_llhttp__internal__n_start_req_60, - s_n_llhttp__internal__n_start_req_65, - s_n_llhttp__internal__n_start_req_67, - s_n_llhttp__internal__n_start_req_68, - s_n_llhttp__internal__n_start_req_66, - s_n_llhttp__internal__n_start_req_69, - s_n_llhttp__internal__n_start_req_64, - s_n_llhttp__internal__n_start_req_63, - s_n_llhttp__internal__n_start_req, - s_n_llhttp__internal__n_invoke_llhttp__on_status_complete, - s_n_llhttp__internal__n_res_line_almost_done, - s_n_llhttp__internal__n_res_status, - s_n_llhttp__internal__n_span_start_llhttp__on_status, - s_n_llhttp__internal__n_res_status_start, - s_n_llhttp__internal__n_res_status_code_otherwise, - s_n_llhttp__internal__n_res_status_code, - s_n_llhttp__internal__n_res_http_end, - s_n_llhttp__internal__n_res_http_minor, - s_n_llhttp__internal__n_res_http_dot, - s_n_llhttp__internal__n_res_http_major, - s_n_llhttp__internal__n_start_res, - s_n_llhttp__internal__n_req_or_res_method_2, - s_n_llhttp__internal__n_req_or_res_method_3, - s_n_llhttp__internal__n_req_or_res_method_1, - s_n_llhttp__internal__n_req_or_res_method, - s_n_llhttp__internal__n_start_req_or_res, - s_n_llhttp__internal__n_invoke_load_type, - s_n_llhttp__internal__n_start, -}; -typedef enum llparse_state_e llparse_state_t; - -int llhttp__on_url( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__on_header_field( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__on_header_value( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__on_body( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__on_status( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__internal__c_update_finish( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->finish = 2; - return 0; -} - -int llhttp__on_message_begin( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__internal__c_load_type( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return state->type; -} - -int llhttp__internal__c_store_method( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp, - int match) { - state->method = match; - return 0; -} - -int llhttp__internal__c_is_equal_method( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return state->method == 5; -} - -int llhttp__internal__c_update_http_major( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->http_major = 0; - return 0; -} - -int llhttp__internal__c_update_http_minor( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->http_minor = 9; - return 0; -} - -int llhttp__on_url_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__internal__c_test_flags( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return (state->flags & 128) == 128; -} - -int llhttp__on_chunk_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__on_message_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__internal__c_is_equal_upgrade( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return state->upgrade == 1; -} - -int llhttp__after_message_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__internal__c_update_finish_1( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->finish = 0; - return 0; -} - -int llhttp__internal__c_test_lenient_flags( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return (state->lenient_flags & 4) == 4; -} - -int llhttp__internal__c_test_flags_1( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return (state->flags & 544) == 544; -} - -int llhttp__internal__c_test_lenient_flags_1( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return (state->lenient_flags & 2) == 2; -} - -int llhttp__before_headers_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__on_headers_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__after_headers_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__internal__c_update_content_length( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->content_length = 0; - return 0; -} - -int llhttp__internal__c_mul_add_content_length( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp, - int match) { - /* Multiplication overflow */ - if (state->content_length > 0xffffffffffffffffULL / 16) { - return 1; - } - - state->content_length *= 16; - - /* Addition overflow */ - if (match >= 0) { - if (state->content_length > 0xffffffffffffffffULL - match) { - return 1; - } - } else { - if (state->content_length < 0ULL - match) { - return 1; - } - } - state->content_length += match; - return 0; -} - -int llhttp__on_chunk_header( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__internal__c_is_equal_content_length( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return state->content_length == 0; -} - -int llhttp__internal__c_or_flags( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->flags |= 128; - return 0; -} - -int llhttp__internal__c_update_finish_3( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->finish = 1; - return 0; -} - -int llhttp__internal__c_or_flags_1( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->flags |= 64; - return 0; -} - -int llhttp__internal__c_update_upgrade( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->upgrade = 1; - return 0; -} - -int llhttp__internal__c_store_header_state( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp, - int match) { - state->header_state = match; - return 0; -} - -int llhttp__on_header_field_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__internal__c_load_header_state( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return state->header_state; -} - -int llhttp__internal__c_or_flags_3( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->flags |= 1; - return 0; -} - -int llhttp__internal__c_update_header_state( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->header_state = 1; - return 0; -} - -int llhttp__on_header_value_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__internal__c_or_flags_4( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->flags |= 2; - return 0; -} - -int llhttp__internal__c_or_flags_5( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->flags |= 4; - return 0; -} - -int llhttp__internal__c_or_flags_6( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->flags |= 8; - return 0; -} - -int llhttp__internal__c_update_header_state_2( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->header_state = 6; - return 0; -} - -int llhttp__internal__c_test_lenient_flags_2( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return (state->lenient_flags & 1) == 1; -} - -int llhttp__internal__c_update_header_state_4( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->header_state = 0; - return 0; -} - -int llhttp__internal__c_update_header_state_5( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->header_state = 5; - return 0; -} - -int llhttp__internal__c_update_header_state_6( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->header_state = 7; - return 0; -} - -int llhttp__internal__c_test_flags_2( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return (state->flags & 32) == 32; -} - -int llhttp__internal__c_mul_add_content_length_1( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp, - int match) { - /* Multiplication overflow */ - if (state->content_length > 0xffffffffffffffffULL / 10) { - return 1; - } - - state->content_length *= 10; - - /* Addition overflow */ - if (match >= 0) { - if (state->content_length > 0xffffffffffffffffULL - match) { - return 1; - } - } else { - if (state->content_length < 0ULL - match) { - return 1; - } - } - state->content_length += match; - return 0; -} - -int llhttp__internal__c_or_flags_15( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->flags |= 32; - return 0; -} - -int llhttp__internal__c_or_flags_16( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->flags |= 512; - return 0; -} - -int llhttp__internal__c_and_flags( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->flags &= -9; - return 0; -} - -int llhttp__internal__c_update_header_state_7( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->header_state = 8; - return 0; -} - -int llhttp__internal__c_or_flags_17( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->flags |= 16; - return 0; -} - -int llhttp__internal__c_load_method( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - return state->method; -} - -int llhttp__internal__c_store_http_major( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp, - int match) { - state->http_major = match; - return 0; -} - -int llhttp__internal__c_store_http_minor( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp, - int match) { - state->http_minor = match; - return 0; -} - -int llhttp__internal__c_update_status_code( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->status_code = 0; - return 0; -} - -int llhttp__internal__c_mul_add_status_code( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp, - int match) { - /* Multiplication overflow */ - if (state->status_code > 0xffff / 10) { - return 1; - } - - state->status_code *= 10; - - /* Addition overflow */ - if (match >= 0) { - if (state->status_code > 0xffff - match) { - return 1; - } - } else { - if (state->status_code < 0 - match) { - return 1; - } - } - state->status_code += match; - - /* Enforce maximum */ - if (state->status_code > 999) { - return 1; - } - return 0; -} - -int llhttp__on_status_complete( - llhttp__internal_t* s, const unsigned char* p, - const unsigned char* endp); - -int llhttp__internal__c_update_type( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->type = 1; - return 0; -} - -int llhttp__internal__c_update_type_1( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - state->type = 2; - return 0; -} - -int llhttp__internal_init(llhttp__internal_t* state) { - memset(state, 0, sizeof(*state)); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_start; - return 0; -} - -static llparse_state_t llhttp__internal__run( - llhttp__internal_t* state, - const unsigned char* p, - const unsigned char* endp) { - int match; - switch ((llparse_state_t) (intptr_t) state->_current) { - case s_n_llhttp__internal__n_closed: - s_n_llhttp__internal__n_closed: { - if (p == endp) { - return s_n_llhttp__internal__n_closed; - } - p++; - goto s_n_llhttp__internal__n_closed; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_invoke_llhttp__after_message_complete: - s_n_llhttp__internal__n_invoke_llhttp__after_message_complete: { - switch (llhttp__after_message_complete(state, p, endp)) { - case 1: - goto s_n_llhttp__internal__n_invoke_update_finish_2; - default: - goto s_n_llhttp__internal__n_invoke_update_finish_1; - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_pause_1: - s_n_llhttp__internal__n_pause_1: { - state->error = 0x16; - state->reason = "Pause on CONNECT/Upgrade"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; - return s_error; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_invoke_is_equal_upgrade: - s_n_llhttp__internal__n_invoke_is_equal_upgrade: { - switch (llhttp__internal__c_is_equal_upgrade(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; - default: - goto s_n_llhttp__internal__n_pause_1; - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2: - s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2: { - switch (llhttp__on_message_complete(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_invoke_is_equal_upgrade; - case 21: - goto s_n_llhttp__internal__n_pause_5; - default: - goto s_n_llhttp__internal__n_error_9; - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_chunk_data_almost_done_skip: - s_n_llhttp__internal__n_chunk_data_almost_done_skip: { - if (p == endp) { - return s_n_llhttp__internal__n_chunk_data_almost_done_skip; - } - p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_chunk_data_almost_done: - s_n_llhttp__internal__n_chunk_data_almost_done: { - if (p == endp) { - return s_n_llhttp__internal__n_chunk_data_almost_done; - } - p++; - goto s_n_llhttp__internal__n_chunk_data_almost_done_skip; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_consume_content_length: - s_n_llhttp__internal__n_consume_content_length: { - size_t avail; - size_t need; - - avail = endp - p; - need = state->content_length; - if (avail >= need) { - p += need; - state->content_length = 0; - goto s_n_llhttp__internal__n_span_end_llhttp__on_body; - } - - state->content_length -= avail; - return s_n_llhttp__internal__n_consume_content_length; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_start_llhttp__on_body: - s_n_llhttp__internal__n_span_start_llhttp__on_body: { - if (p == endp) { - return s_n_llhttp__internal__n_span_start_llhttp__on_body; - } - state->_span_pos0 = (void*) p; - state->_span_cb0 = llhttp__on_body; - goto s_n_llhttp__internal__n_consume_content_length; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_invoke_is_equal_content_length: - s_n_llhttp__internal__n_invoke_is_equal_content_length: { - switch (llhttp__internal__c_is_equal_content_length(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_span_start_llhttp__on_body; - default: - goto s_n_llhttp__internal__n_invoke_or_flags; - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_chunk_size_almost_done: - s_n_llhttp__internal__n_chunk_size_almost_done: { - if (p == endp) { - return s_n_llhttp__internal__n_chunk_size_almost_done; - } - p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_header; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_chunk_parameters: - s_n_llhttp__internal__n_chunk_parameters: { - if (p == endp) { - return s_n_llhttp__internal__n_chunk_parameters; - } - switch (*p) { - case 13: { - p++; - goto s_n_llhttp__internal__n_chunk_size_almost_done; - } - default: { - p++; - goto s_n_llhttp__internal__n_chunk_parameters; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_chunk_size_otherwise: - s_n_llhttp__internal__n_chunk_size_otherwise: { - if (p == endp) { - return s_n_llhttp__internal__n_chunk_size_otherwise; - } - switch (*p) { - case 13: { - p++; - goto s_n_llhttp__internal__n_chunk_size_almost_done; - } - case ' ': { - p++; - goto s_n_llhttp__internal__n_chunk_parameters; - } - case ';': { - p++; - goto s_n_llhttp__internal__n_chunk_parameters; - } - default: { - goto s_n_llhttp__internal__n_error_6; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_chunk_size: - s_n_llhttp__internal__n_chunk_size: { - if (p == endp) { - return s_n_llhttp__internal__n_chunk_size; - } - switch (*p) { - case '0': { - p++; - match = 0; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '1': { - p++; - match = 1; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '2': { - p++; - match = 2; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '3': { - p++; - match = 3; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '4': { - p++; - match = 4; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '5': { - p++; - match = 5; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '6': { - p++; - match = 6; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '7': { - p++; - match = 7; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '8': { - p++; - match = 8; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '9': { - p++; - match = 9; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'A': { - p++; - match = 10; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'B': { - p++; - match = 11; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'C': { - p++; - match = 12; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'D': { - p++; - match = 13; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'E': { - p++; - match = 14; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'F': { - p++; - match = 15; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'a': { - p++; - match = 10; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'b': { - p++; - match = 11; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'c': { - p++; - match = 12; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'd': { - p++; - match = 13; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'e': { - p++; - match = 14; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'f': { - p++; - match = 15; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - default: { - goto s_n_llhttp__internal__n_chunk_size_otherwise; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_chunk_size_digit: - s_n_llhttp__internal__n_chunk_size_digit: { - if (p == endp) { - return s_n_llhttp__internal__n_chunk_size_digit; - } - switch (*p) { - case '0': { - p++; - match = 0; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '1': { - p++; - match = 1; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '2': { - p++; - match = 2; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '3': { - p++; - match = 3; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '4': { - p++; - match = 4; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '5': { - p++; - match = 5; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '6': { - p++; - match = 6; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '7': { - p++; - match = 7; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '8': { - p++; - match = 8; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case '9': { - p++; - match = 9; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'A': { - p++; - match = 10; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'B': { - p++; - match = 11; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'C': { - p++; - match = 12; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'D': { - p++; - match = 13; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'E': { - p++; - match = 14; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'F': { - p++; - match = 15; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'a': { - p++; - match = 10; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'b': { - p++; - match = 11; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'c': { - p++; - match = 12; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'd': { - p++; - match = 13; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'e': { - p++; - match = 14; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - case 'f': { - p++; - match = 15; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length; - } - default: { - goto s_n_llhttp__internal__n_error_8; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_invoke_update_content_length: - s_n_llhttp__internal__n_invoke_update_content_length: { - switch (llhttp__internal__c_update_content_length(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_chunk_size_digit; - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_consume_content_length_1: - s_n_llhttp__internal__n_consume_content_length_1: { - size_t avail; - size_t need; - - avail = endp - p; - need = state->content_length; - if (avail >= need) { - p += need; - state->content_length = 0; - goto s_n_llhttp__internal__n_span_end_llhttp__on_body_1; - } - - state->content_length -= avail; - return s_n_llhttp__internal__n_consume_content_length_1; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_start_llhttp__on_body_1: - s_n_llhttp__internal__n_span_start_llhttp__on_body_1: { - if (p == endp) { - return s_n_llhttp__internal__n_span_start_llhttp__on_body_1; - } - state->_span_pos0 = (void*) p; - state->_span_cb0 = llhttp__on_body; - goto s_n_llhttp__internal__n_consume_content_length_1; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_eof: - s_n_llhttp__internal__n_eof: { - if (p == endp) { - return s_n_llhttp__internal__n_eof; - } - p++; - goto s_n_llhttp__internal__n_eof; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_start_llhttp__on_body_2: - s_n_llhttp__internal__n_span_start_llhttp__on_body_2: { - if (p == endp) { - return s_n_llhttp__internal__n_span_start_llhttp__on_body_2; - } - state->_span_pos0 = (void*) p; - state->_span_cb0 = llhttp__on_body; - goto s_n_llhttp__internal__n_eof; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete: - s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete: { - switch (llhttp__after_headers_complete(state, p, endp)) { - case 1: - goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_1; - case 2: - goto s_n_llhttp__internal__n_invoke_update_content_length; - case 3: - goto s_n_llhttp__internal__n_span_start_llhttp__on_body_1; - case 4: - goto s_n_llhttp__internal__n_invoke_update_finish_3; - case 5: - goto s_n_llhttp__internal__n_error_10; - default: - goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete; - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_headers_almost_done: - s_n_llhttp__internal__n_headers_almost_done: { - if (p == endp) { - return s_n_llhttp__internal__n_headers_almost_done; - } - p++; - goto s_n_llhttp__internal__n_invoke_test_flags; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete: - s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete: { - switch (llhttp__on_header_value_complete(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_field_start; - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_start_llhttp__on_header_value: - s_n_llhttp__internal__n_span_start_llhttp__on_header_value: { - if (p == endp) { - return s_n_llhttp__internal__n_span_start_llhttp__on_header_value; - } - state->_span_pos0 = (void*) p; - state->_span_cb0 = llhttp__on_header_value; - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_discard_lws: - s_n_llhttp__internal__n_header_value_discard_lws: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_discard_lws; - } - switch (*p) { - case 9: { - p++; - goto s_n_llhttp__internal__n_header_value_discard_ws; - } - case ' ': { - p++; - goto s_n_llhttp__internal__n_header_value_discard_ws; - } - default: { - goto s_n_llhttp__internal__n_invoke_load_header_state; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_discard_ws_almost_done: - s_n_llhttp__internal__n_header_value_discard_ws_almost_done: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_discard_ws_almost_done; - } - p++; - goto s_n_llhttp__internal__n_header_value_discard_lws; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_lws: - s_n_llhttp__internal__n_header_value_lws: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_lws; - } - switch (*p) { - case 9: { - goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; - } - case ' ': { - goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; - } - default: { - goto s_n_llhttp__internal__n_invoke_load_header_state_3; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_almost_done: - s_n_llhttp__internal__n_header_value_almost_done: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_almost_done; - } - switch (*p) { - case 10: { - p++; - goto s_n_llhttp__internal__n_header_value_lws; - } - default: { - goto s_n_llhttp__internal__n_error_14; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_lenient: - s_n_llhttp__internal__n_header_value_lenient: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_lenient; - } - switch (*p) { - case 10: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1; - } - case 13: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3; - } - default: { - p++; - goto s_n_llhttp__internal__n_header_value_lenient; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_otherwise: - s_n_llhttp__internal__n_header_value_otherwise: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_otherwise; - } - switch (*p) { - case 10: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1; - } - case 13: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2; - } - default: { - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_2; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_connection_token: - s_n_llhttp__internal__n_header_value_connection_token: { - static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 - }; - if (p == endp) { - return s_n_llhttp__internal__n_header_value_connection_token; - } - switch (lookup_table[(uint8_t) *p]) { - case 1: { - p++; - goto s_n_llhttp__internal__n_header_value_connection_token; - } - case 2: { - p++; - goto s_n_llhttp__internal__n_header_value_connection; - } - default: { - goto s_n_llhttp__internal__n_header_value_otherwise; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_connection_ws: - s_n_llhttp__internal__n_header_value_connection_ws: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_connection_ws; - } - switch (*p) { - case 10: { - goto s_n_llhttp__internal__n_header_value_otherwise; - } - case 13: { - goto s_n_llhttp__internal__n_header_value_otherwise; - } - case ' ': { - p++; - goto s_n_llhttp__internal__n_header_value_connection_ws; - } - case ',': { - p++; - goto s_n_llhttp__internal__n_invoke_load_header_state_4; - } - default: { - goto s_n_llhttp__internal__n_invoke_update_header_state_4; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_connection_1: - s_n_llhttp__internal__n_header_value_connection_1: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_header_value_connection_1; - } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob3, 4); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_invoke_update_header_state_2; - } - case kMatchPause: { - return s_n_llhttp__internal__n_header_value_connection_1; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_header_value_connection_token; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_connection_2: - s_n_llhttp__internal__n_header_value_connection_2: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_header_value_connection_2; - } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob4, 9); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_invoke_update_header_state_5; - } - case kMatchPause: { - return s_n_llhttp__internal__n_header_value_connection_2; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_header_value_connection_token; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_connection_3: - s_n_llhttp__internal__n_header_value_connection_3: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_header_value_connection_3; - } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob5, 6); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_invoke_update_header_state_6; - } - case kMatchPause: { - return s_n_llhttp__internal__n_header_value_connection_3; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_header_value_connection_token; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_connection: - s_n_llhttp__internal__n_header_value_connection: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_connection; - } - switch (((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p))) { - case 9: { - p++; - goto s_n_llhttp__internal__n_header_value_connection; - } - case ' ': { - p++; - goto s_n_llhttp__internal__n_header_value_connection; - } - case 'c': { - p++; - goto s_n_llhttp__internal__n_header_value_connection_1; - } - case 'k': { - p++; - goto s_n_llhttp__internal__n_header_value_connection_2; - } - case 'u': { - p++; - goto s_n_llhttp__internal__n_header_value_connection_3; - } - default: { - goto s_n_llhttp__internal__n_header_value_connection_token; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_error_17: - s_n_llhttp__internal__n_error_17: { - state->error = 0xb; - state->reason = "Content-Length overflow"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_error_18: - s_n_llhttp__internal__n_error_18: { - state->error = 0xb; - state->reason = "Invalid character in Content-Length"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_content_length_ws: - s_n_llhttp__internal__n_header_value_content_length_ws: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_content_length_ws; - } - switch (*p) { - case 10: { - goto s_n_llhttp__internal__n_invoke_or_flags_15; - } - case 13: { - goto s_n_llhttp__internal__n_invoke_or_flags_15; - } - case ' ': { - p++; - goto s_n_llhttp__internal__n_header_value_content_length_ws; - } - default: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_content_length: - s_n_llhttp__internal__n_header_value_content_length: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_content_length; - } - switch (*p) { - case '0': { - p++; - match = 0; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; - } - case '1': { - p++; - match = 1; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; - } - case '2': { - p++; - match = 2; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; - } - case '3': { - p++; - match = 3; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; - } - case '4': { - p++; - match = 4; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; - } - case '5': { - p++; - match = 5; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; - } - case '6': { - p++; - match = 6; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; - } - case '7': { - p++; - match = 7; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; - } - case '8': { - p++; - match = 8; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; - } - case '9': { - p++; - match = 9; - goto s_n_llhttp__internal__n_invoke_mul_add_content_length_1; - } - default: { - goto s_n_llhttp__internal__n_header_value_content_length_ws; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_te_chunked_last: - s_n_llhttp__internal__n_header_value_te_chunked_last: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_te_chunked_last; - } - switch (*p) { - case 10: { - goto s_n_llhttp__internal__n_invoke_update_header_state_7; - } - case 13: { - goto s_n_llhttp__internal__n_invoke_update_header_state_7; - } - case ' ': { - p++; - goto s_n_llhttp__internal__n_header_value_te_chunked_last; - } - default: { - goto s_n_llhttp__internal__n_header_value_te_chunked; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_te_token_ows: - s_n_llhttp__internal__n_header_value_te_token_ows: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_te_token_ows; - } - switch (*p) { - case 9: { - p++; - goto s_n_llhttp__internal__n_header_value_te_token_ows; - } - case ' ': { - p++; - goto s_n_llhttp__internal__n_header_value_te_token_ows; - } - default: { - goto s_n_llhttp__internal__n_header_value_te_chunked; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value: - s_n_llhttp__internal__n_header_value: { - static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 - }; - if (p == endp) { - return s_n_llhttp__internal__n_header_value; - } - #ifdef __SSE4_2__ - if (endp - p >= 16) { - __m128i ranges; - __m128i input; - int avail; - int match_len; - - /* Load input */ - input = _mm_loadu_si128((__m128i const*) p); - ranges = _mm_loadu_si128((__m128i const*) llparse_blob7); - - /* Find first character that does not match `ranges` */ - match_len = _mm_cmpestri(ranges, 6, - input, 16, - _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | - _SIDD_NEGATIVE_POLARITY); - - if (match_len != 0) { - p += match_len; - goto s_n_llhttp__internal__n_header_value; - } - goto s_n_llhttp__internal__n_header_value_otherwise; - } - #endif /* __SSE4_2__ */ - switch (lookup_table[(uint8_t) *p]) { - case 1: { - p++; - goto s_n_llhttp__internal__n_header_value; - } - default: { - goto s_n_llhttp__internal__n_header_value_otherwise; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_te_token: - s_n_llhttp__internal__n_header_value_te_token: { - static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 - }; - if (p == endp) { - return s_n_llhttp__internal__n_header_value_te_token; - } - switch (lookup_table[(uint8_t) *p]) { - case 1: { - p++; - goto s_n_llhttp__internal__n_header_value_te_token; - } - case 2: { - p++; - goto s_n_llhttp__internal__n_header_value_te_token_ows; - } - default: { - goto s_n_llhttp__internal__n_invoke_update_header_state_8; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_te_chunked: - s_n_llhttp__internal__n_header_value_te_chunked: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_header_value_te_chunked; - } - match_seq = llparse__match_sequence_to_lower_unsafe(state, p, endp, llparse_blob6, 7); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_header_value_te_chunked_last; - } - case kMatchPause: { - return s_n_llhttp__internal__n_header_value_te_chunked; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_header_value_te_token; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1: - s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1: { - if (p == endp) { - return s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; - } - state->_span_pos0 = (void*) p; - state->_span_cb0 = llhttp__on_header_value; - goto s_n_llhttp__internal__n_invoke_load_header_state_2; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_value_discard_ws: - s_n_llhttp__internal__n_header_value_discard_ws: { - if (p == endp) { - return s_n_llhttp__internal__n_header_value_discard_ws; - } - switch (*p) { - case 9: { - p++; - goto s_n_llhttp__internal__n_header_value_discard_ws; - } - case 10: { - p++; - goto s_n_llhttp__internal__n_header_value_discard_lws; - } - case 13: { - p++; - goto s_n_llhttp__internal__n_header_value_discard_ws_almost_done; - } - case ' ': { - p++; - goto s_n_llhttp__internal__n_header_value_discard_ws; - } - default: { - goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete: - s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete: { - switch (llhttp__on_header_field_complete(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_value_discard_ws; - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_field_general_otherwise: - s_n_llhttp__internal__n_header_field_general_otherwise: { - if (p == endp) { - return s_n_llhttp__internal__n_header_field_general_otherwise; - } - switch (*p) { - case ':': { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field_1; - } - default: { - goto s_n_llhttp__internal__n_error_19; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_field_general: - s_n_llhttp__internal__n_header_field_general: { - static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - if (p == endp) { - return s_n_llhttp__internal__n_header_field_general; - } - #ifdef __SSE4_2__ - if (endp - p >= 16) { - __m128i ranges; - __m128i input; - int avail; - int match_len; - - /* Load input */ - input = _mm_loadu_si128((__m128i const*) p); - ranges = _mm_loadu_si128((__m128i const*) llparse_blob8); - - /* Find first character that does not match `ranges` */ - match_len = _mm_cmpestri(ranges, 16, - input, 16, - _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | - _SIDD_NEGATIVE_POLARITY); - - if (match_len != 0) { - p += match_len; - goto s_n_llhttp__internal__n_header_field_general; - } - ranges = _mm_loadu_si128((__m128i const*) llparse_blob9); - - /* Find first character that does not match `ranges` */ - match_len = _mm_cmpestri(ranges, 2, - input, 16, - _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | - _SIDD_NEGATIVE_POLARITY); - - if (match_len != 0) { - p += match_len; - goto s_n_llhttp__internal__n_header_field_general; - } - goto s_n_llhttp__internal__n_header_field_general_otherwise; - } - #endif /* __SSE4_2__ */ - switch (lookup_table[(uint8_t) *p]) { - case 1: { - p++; - goto s_n_llhttp__internal__n_header_field_general; - } - default: { - goto s_n_llhttp__internal__n_header_field_general_otherwise; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_field_colon: - s_n_llhttp__internal__n_header_field_colon: { - if (p == endp) { - return s_n_llhttp__internal__n_header_field_colon; - } - switch (*p) { - case ' ': { - p++; - goto s_n_llhttp__internal__n_header_field_colon; - } - case ':': { - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field; - } - default: { - goto s_n_llhttp__internal__n_invoke_update_header_state_9; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_field_3: - s_n_llhttp__internal__n_header_field_3: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_header_field_3; - } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob2, 6); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 1; - goto s_n_llhttp__internal__n_invoke_store_header_state; - } - case kMatchPause: { - return s_n_llhttp__internal__n_header_field_3; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_field_4: - s_n_llhttp__internal__n_header_field_4: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_header_field_4; - } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob10, 10); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 2; - goto s_n_llhttp__internal__n_invoke_store_header_state; - } - case kMatchPause: { - return s_n_llhttp__internal__n_header_field_4; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_field_2: - s_n_llhttp__internal__n_header_field_2: { - if (p == endp) { - return s_n_llhttp__internal__n_header_field_2; - } - switch (((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p))) { - case 'n': { - p++; - goto s_n_llhttp__internal__n_header_field_3; - } - case 't': { - p++; - goto s_n_llhttp__internal__n_header_field_4; - } - default: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_field_1: - s_n_llhttp__internal__n_header_field_1: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_header_field_1; - } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob1, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_header_field_2; - } - case kMatchPause: { - return s_n_llhttp__internal__n_header_field_1; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_field_5: - s_n_llhttp__internal__n_header_field_5: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_header_field_5; - } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob11, 15); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 1; - goto s_n_llhttp__internal__n_invoke_store_header_state; - } - case kMatchPause: { - return s_n_llhttp__internal__n_header_field_5; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_field_6: - s_n_llhttp__internal__n_header_field_6: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_header_field_6; - } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob12, 16); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 3; - goto s_n_llhttp__internal__n_invoke_store_header_state; - } - case kMatchPause: { - return s_n_llhttp__internal__n_header_field_6; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_field_7: - s_n_llhttp__internal__n_header_field_7: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_header_field_7; - } - match_seq = llparse__match_sequence_to_lower(state, p, endp, llparse_blob13, 6); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - match = 4; - goto s_n_llhttp__internal__n_invoke_store_header_state; - } - case kMatchPause: { - return s_n_llhttp__internal__n_header_field_7; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_field: - s_n_llhttp__internal__n_header_field: { - if (p == endp) { - return s_n_llhttp__internal__n_header_field; - } - switch (((*p) >= 'A' && (*p) <= 'Z' ? (*p | 0x20) : (*p))) { - case 'c': { - p++; - goto s_n_llhttp__internal__n_header_field_1; - } - case 'p': { - p++; - goto s_n_llhttp__internal__n_header_field_5; - } - case 't': { - p++; - goto s_n_llhttp__internal__n_header_field_6; - } - case 'u': { - p++; - goto s_n_llhttp__internal__n_header_field_7; - } - default: { - goto s_n_llhttp__internal__n_invoke_update_header_state_10; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_start_llhttp__on_header_field: - s_n_llhttp__internal__n_span_start_llhttp__on_header_field: { - if (p == endp) { - return s_n_llhttp__internal__n_span_start_llhttp__on_header_field; - } - state->_span_pos0 = (void*) p; - state->_span_cb0 = llhttp__on_header_field; - goto s_n_llhttp__internal__n_header_field; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_header_field_start: - s_n_llhttp__internal__n_header_field_start: { - if (p == endp) { - return s_n_llhttp__internal__n_header_field_start; - } - switch (*p) { - case 10: { - goto s_n_llhttp__internal__n_headers_almost_done; - } - case 13: { - p++; - goto s_n_llhttp__internal__n_headers_almost_done; - } - default: { - goto s_n_llhttp__internal__n_span_start_llhttp__on_header_field; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_skip_to_http09: - s_n_llhttp__internal__n_url_skip_to_http09: { - if (p == endp) { - return s_n_llhttp__internal__n_url_skip_to_http09; - } - p++; - goto s_n_llhttp__internal__n_invoke_update_http_major; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_skip_lf_to_http09: - s_n_llhttp__internal__n_url_skip_lf_to_http09: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_url_skip_lf_to_http09; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob14, 2); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_invoke_update_http_major; - } - case kMatchPause: { - return s_n_llhttp__internal__n_url_skip_lf_to_http09; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_20; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_pri_upgrade: - s_n_llhttp__internal__n_req_pri_upgrade: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_req_pri_upgrade; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob16, 10); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_error_23; - } - case kMatchPause: { - return s_n_llhttp__internal__n_req_pri_upgrade; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_24; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_http_complete_1: - s_n_llhttp__internal__n_req_http_complete_1: { - if (p == endp) { - return s_n_llhttp__internal__n_req_http_complete_1; - } - switch (*p) { - case 10: { - p++; - goto s_n_llhttp__internal__n_header_field_start; - } - default: { - goto s_n_llhttp__internal__n_error_22; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_http_complete: - s_n_llhttp__internal__n_req_http_complete: { - if (p == endp) { - return s_n_llhttp__internal__n_req_http_complete; - } - switch (*p) { - case 10: { - p++; - goto s_n_llhttp__internal__n_header_field_start; - } - case 13: { - p++; - goto s_n_llhttp__internal__n_req_http_complete_1; - } - default: { - goto s_n_llhttp__internal__n_error_22; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_http_minor: - s_n_llhttp__internal__n_req_http_minor: { - if (p == endp) { - return s_n_llhttp__internal__n_req_http_minor; - } - switch (*p) { - case '0': { - p++; - match = 0; - goto s_n_llhttp__internal__n_invoke_store_http_minor; - } - case '1': { - p++; - match = 1; - goto s_n_llhttp__internal__n_invoke_store_http_minor; - } - case '2': { - p++; - match = 2; - goto s_n_llhttp__internal__n_invoke_store_http_minor; - } - case '3': { - p++; - match = 3; - goto s_n_llhttp__internal__n_invoke_store_http_minor; - } - case '4': { - p++; - match = 4; - goto s_n_llhttp__internal__n_invoke_store_http_minor; - } - case '5': { - p++; - match = 5; - goto s_n_llhttp__internal__n_invoke_store_http_minor; - } - case '6': { - p++; - match = 6; - goto s_n_llhttp__internal__n_invoke_store_http_minor; - } - case '7': { - p++; - match = 7; - goto s_n_llhttp__internal__n_invoke_store_http_minor; - } - case '8': { - p++; - match = 8; - goto s_n_llhttp__internal__n_invoke_store_http_minor; - } - case '9': { - p++; - match = 9; - goto s_n_llhttp__internal__n_invoke_store_http_minor; - } - default: { - goto s_n_llhttp__internal__n_error_25; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_http_dot: - s_n_llhttp__internal__n_req_http_dot: { - if (p == endp) { - return s_n_llhttp__internal__n_req_http_dot; - } - switch (*p) { - case '.': { - p++; - goto s_n_llhttp__internal__n_req_http_minor; - } - default: { - goto s_n_llhttp__internal__n_error_26; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_http_major: - s_n_llhttp__internal__n_req_http_major: { - if (p == endp) { - return s_n_llhttp__internal__n_req_http_major; - } - switch (*p) { - case '0': { - p++; - match = 0; - goto s_n_llhttp__internal__n_invoke_store_http_major; - } - case '1': { - p++; - match = 1; - goto s_n_llhttp__internal__n_invoke_store_http_major; - } - case '2': { - p++; - match = 2; - goto s_n_llhttp__internal__n_invoke_store_http_major; - } - case '3': { - p++; - match = 3; - goto s_n_llhttp__internal__n_invoke_store_http_major; - } - case '4': { - p++; - match = 4; - goto s_n_llhttp__internal__n_invoke_store_http_major; - } - case '5': { - p++; - match = 5; - goto s_n_llhttp__internal__n_invoke_store_http_major; - } - case '6': { - p++; - match = 6; - goto s_n_llhttp__internal__n_invoke_store_http_major; - } - case '7': { - p++; - match = 7; - goto s_n_llhttp__internal__n_invoke_store_http_major; - } - case '8': { - p++; - match = 8; - goto s_n_llhttp__internal__n_invoke_store_http_major; - } - case '9': { - p++; - match = 9; - goto s_n_llhttp__internal__n_invoke_store_http_major; - } - default: { - goto s_n_llhttp__internal__n_error_27; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_http_start_1: - s_n_llhttp__internal__n_req_http_start_1: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_req_http_start_1; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob15, 4); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_invoke_load_method; - } - case kMatchPause: { - return s_n_llhttp__internal__n_req_http_start_1; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_30; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_http_start_2: - s_n_llhttp__internal__n_req_http_start_2: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_req_http_start_2; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob17, 3); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_invoke_load_method_2; - } - case kMatchPause: { - return s_n_llhttp__internal__n_req_http_start_2; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_30; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_http_start_3: - s_n_llhttp__internal__n_req_http_start_3: { - llparse_match_t match_seq; - - if (p == endp) { - return s_n_llhttp__internal__n_req_http_start_3; - } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob18, 4); - p = match_seq.current; - switch (match_seq.status) { - case kMatchComplete: { - p++; - goto s_n_llhttp__internal__n_invoke_load_method_3; - } - case kMatchPause: { - return s_n_llhttp__internal__n_req_http_start_3; - } - case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_30; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_req_http_start: - s_n_llhttp__internal__n_req_http_start: { - if (p == endp) { - return s_n_llhttp__internal__n_req_http_start; - } - switch (*p) { - case ' ': { - p++; - goto s_n_llhttp__internal__n_req_http_start; - } - case 'H': { - p++; - goto s_n_llhttp__internal__n_req_http_start_1; - } - case 'I': { - p++; - goto s_n_llhttp__internal__n_req_http_start_2; - } - case 'R': { - p++; - goto s_n_llhttp__internal__n_req_http_start_3; - } - default: { - goto s_n_llhttp__internal__n_error_30; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_skip_to_http: - s_n_llhttp__internal__n_url_skip_to_http: { - if (p == endp) { - return s_n_llhttp__internal__n_url_skip_to_http; - } - p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_url_complete_1; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_fragment: - s_n_llhttp__internal__n_url_fragment: { - static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 3, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 - }; - if (p == endp) { - return s_n_llhttp__internal__n_url_fragment; - } - switch (lookup_table[(uint8_t) *p]) { - case 1: { - p++; - goto s_n_llhttp__internal__n_url_fragment; - } - case 2: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_6; - } - case 3: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_7; - } - case 4: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_8; - } - default: { - goto s_n_llhttp__internal__n_error_31; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_end_stub_query_3: - s_n_llhttp__internal__n_span_end_stub_query_3: { - if (p == endp) { - return s_n_llhttp__internal__n_span_end_stub_query_3; - } - p++; - goto s_n_llhttp__internal__n_url_fragment; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_query: - s_n_llhttp__internal__n_url_query: { - static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 1, 3, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 4, 1, 1, 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 - }; - if (p == endp) { - return s_n_llhttp__internal__n_url_query; - } - switch (lookup_table[(uint8_t) *p]) { - case 1: { - p++; - goto s_n_llhttp__internal__n_url_query; - } - case 2: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_9; - } - case 3: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_10; - } - case 4: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_11; - } - case 5: { - goto s_n_llhttp__internal__n_span_end_stub_query_3; - } - default: { - goto s_n_llhttp__internal__n_error_32; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_query_or_fragment: - s_n_llhttp__internal__n_url_query_or_fragment: { - if (p == endp) { - return s_n_llhttp__internal__n_url_query_or_fragment; - } - switch (*p) { - case 10: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_3; - } - case 13: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_4; - } - case ' ': { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_5; - } - case '#': { - p++; - goto s_n_llhttp__internal__n_url_fragment; - } - case '?': { - p++; - goto s_n_llhttp__internal__n_url_query; - } - default: { - goto s_n_llhttp__internal__n_error_33; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_path: - s_n_llhttp__internal__n_url_path: { - static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 - }; - if (p == endp) { - return s_n_llhttp__internal__n_url_path; - } - #ifdef __SSE4_2__ - if (endp - p >= 16) { - __m128i ranges; - __m128i input; - int avail; - int match_len; - - /* Load input */ - input = _mm_loadu_si128((__m128i const*) p); - ranges = _mm_loadu_si128((__m128i const*) llparse_blob0); - - /* Find first character that does not match `ranges` */ - match_len = _mm_cmpestri(ranges, 12, - input, 16, - _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | - _SIDD_NEGATIVE_POLARITY); - - if (match_len != 0) { - p += match_len; - goto s_n_llhttp__internal__n_url_path; - } - goto s_n_llhttp__internal__n_url_query_or_fragment; - } - #endif /* __SSE4_2__ */ - switch (lookup_table[(uint8_t) *p]) { - case 1: { - p++; - goto s_n_llhttp__internal__n_url_path; - } - default: { - goto s_n_llhttp__internal__n_url_query_or_fragment; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_start_stub_path_2: - s_n_llhttp__internal__n_span_start_stub_path_2: { - if (p == endp) { - return s_n_llhttp__internal__n_span_start_stub_path_2; - } - p++; - goto s_n_llhttp__internal__n_url_path; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_start_stub_path: - s_n_llhttp__internal__n_span_start_stub_path: { - if (p == endp) { - return s_n_llhttp__internal__n_span_start_stub_path; - } - p++; - goto s_n_llhttp__internal__n_url_path; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_start_stub_path_1: - s_n_llhttp__internal__n_span_start_stub_path_1: { - if (p == endp) { - return s_n_llhttp__internal__n_span_start_stub_path_1; - } - p++; - goto s_n_llhttp__internal__n_url_path; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_server_with_at: - s_n_llhttp__internal__n_url_server_with_at: { - static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 3, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 0, 6, - 7, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 0, 4, - 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - if (p == endp) { - return s_n_llhttp__internal__n_url_server_with_at; - } - switch (lookup_table[(uint8_t) *p]) { - case 1: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_12; - } - case 2: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_13; - } - case 3: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_14; - } - case 4: { - p++; - goto s_n_llhttp__internal__n_url_server; - } - case 5: { - goto s_n_llhttp__internal__n_span_start_stub_path_1; - } - case 6: { - p++; - goto s_n_llhttp__internal__n_url_query; - } - case 7: { - p++; - goto s_n_llhttp__internal__n_error_34; - } - default: { - goto s_n_llhttp__internal__n_error_35; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_server: - s_n_llhttp__internal__n_url_server: { - static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 3, 4, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 0, 6, - 7, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 0, 4, - 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 4, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - if (p == endp) { - return s_n_llhttp__internal__n_url_server; - } - switch (lookup_table[(uint8_t) *p]) { - case 1: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url; - } - case 2: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_1; - } - case 3: { - goto s_n_llhttp__internal__n_span_end_llhttp__on_url_2; - } - case 4: { - p++; - goto s_n_llhttp__internal__n_url_server; - } - case 5: { - goto s_n_llhttp__internal__n_span_start_stub_path; - } - case 6: { - p++; - goto s_n_llhttp__internal__n_url_query; - } - case 7: { - p++; - goto s_n_llhttp__internal__n_url_server_with_at; - } - default: { - goto s_n_llhttp__internal__n_error_36; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_schema_delim_1: - s_n_llhttp__internal__n_url_schema_delim_1: { - if (p == endp) { - return s_n_llhttp__internal__n_url_schema_delim_1; - } - switch (*p) { - case '/': { - p++; - goto s_n_llhttp__internal__n_url_server; - } - default: { - goto s_n_llhttp__internal__n_error_38; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_schema_delim: - s_n_llhttp__internal__n_url_schema_delim: { - if (p == endp) { - return s_n_llhttp__internal__n_url_schema_delim; - } - switch (*p) { - case 10: { - p++; - goto s_n_llhttp__internal__n_error_37; - } - case 13: { - p++; - goto s_n_llhttp__internal__n_error_37; - } - case ' ': { - p++; - goto s_n_llhttp__internal__n_error_37; - } - case '/': { - p++; - goto s_n_llhttp__internal__n_url_schema_delim_1; - } - default: { - goto s_n_llhttp__internal__n_error_38; - } - } - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_span_end_stub_schema: - s_n_llhttp__internal__n_span_end_stub_schema: { - if (p == endp) { - return s_n_llhttp__internal__n_span_end_stub_schema; - } - p++; - goto s_n_llhttp__internal__n_url_schema_delim; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_url_schema: - s_n_llhttp__internal__n_url_schema: { - static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, @@ -10818,7 +4304,7 @@ static llparse_state_t llhttp__internal__run( switch (lookup_table[(uint8_t) *p]) { case 1: { p++; - goto s_n_llhttp__internal__n_error_37; + goto s_n_llhttp__internal__n_error_2; } case 2: { goto s_n_llhttp__internal__n_span_end_stub_schema; @@ -10828,16 +4314,15 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_schema; } default: { - goto s_n_llhttp__internal__n_error_39; + goto s_n_llhttp__internal__n_error_90; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_url_start: s_n_llhttp__internal__n_url_start: { static uint8_t lookup_table[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -10860,7 +4345,7 @@ static llparse_state_t llhttp__internal__run( switch (lookup_table[(uint8_t) *p]) { case 1: { p++; - goto s_n_llhttp__internal__n_error_37; + goto s_n_llhttp__internal__n_error_2; } case 2: { goto s_n_llhttp__internal__n_span_start_stub_path_2; @@ -10869,11 +4354,10 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_url_schema; } default: { - goto s_n_llhttp__internal__n_error_40; + goto s_n_llhttp__internal__n_error_91; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_start_llhttp__on_url_1: s_n_llhttp__internal__n_span_start_llhttp__on_url_1: { @@ -10883,8 +4367,27 @@ static llparse_state_t llhttp__internal__run( state->_span_pos0 = (void*) p; state->_span_cb0 = llhttp__on_url; goto s_n_llhttp__internal__n_url_start; - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_url_entry_normal: + s_n_llhttp__internal__n_url_entry_normal: { + if (p == endp) { + return s_n_llhttp__internal__n_url_entry_normal; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_error_2; + } + case 12: { + p++; + goto s_n_llhttp__internal__n_error_2; + } + default: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_url_1; + } + } + UNREACHABLE; } case s_n_llhttp__internal__n_span_start_llhttp__on_url: s_n_llhttp__internal__n_span_start_llhttp__on_url: { @@ -10894,8 +4397,27 @@ static llparse_state_t llhttp__internal__run( state->_span_pos0 = (void*) p; state->_span_cb0 = llhttp__on_url; goto s_n_llhttp__internal__n_url_server; - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_url_entry_connect: + s_n_llhttp__internal__n_url_entry_connect: { + if (p == endp) { + return s_n_llhttp__internal__n_url_entry_connect; + } + switch (*p) { + case 9: { + p++; + goto s_n_llhttp__internal__n_error_2; + } + case 12: { + p++; + goto s_n_llhttp__internal__n_error_2; + } + default: { + goto s_n_llhttp__internal__n_span_start_llhttp__on_url; + } + } + UNREACHABLE; } case s_n_llhttp__internal__n_req_spaces_before_url: s_n_llhttp__internal__n_req_spaces_before_url: { @@ -10911,8 +4433,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_is_equal_method; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_req_first_space_before_url: s_n_llhttp__internal__n_req_first_space_before_url: { @@ -10925,16 +4446,27 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_req_spaces_before_url; } default: { - goto s_n_llhttp__internal__n_error_41; + goto s_n_llhttp__internal__n_error_92; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_2: - s_n_llhttp__internal__n_start_req_2: { + case s_n_llhttp__internal__n_invoke_llhttp__on_method_complete_1: + s_n_llhttp__internal__n_invoke_llhttp__on_method_complete_1: { + switch (llhttp__on_method_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_req_first_space_before_url; + case 21: + goto s_n_llhttp__internal__n_pause_29; + default: + goto s_n_llhttp__internal__n_error_111; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_after_start_req_2: + s_n_llhttp__internal__n_after_start_req_2: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_2; + return s_n_llhttp__internal__n_after_start_req_2; } switch (*p) { case 'L': { @@ -10943,20 +4475,19 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_3: - s_n_llhttp__internal__n_start_req_3: { + case s_n_llhttp__internal__n_after_start_req_3: + s_n_llhttp__internal__n_after_start_req_3: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_3; + return s_n_llhttp__internal__n_after_start_req_3; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob19, 6); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob17, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -10965,44 +4496,42 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_3; + return s_n_llhttp__internal__n_after_start_req_3; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_1: - s_n_llhttp__internal__n_start_req_1: { + case s_n_llhttp__internal__n_after_start_req_1: + s_n_llhttp__internal__n_after_start_req_1: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_1; + return s_n_llhttp__internal__n_after_start_req_1; } switch (*p) { case 'C': { p++; - goto s_n_llhttp__internal__n_start_req_2; + goto s_n_llhttp__internal__n_after_start_req_2; } case 'N': { p++; - goto s_n_llhttp__internal__n_start_req_3; + goto s_n_llhttp__internal__n_after_start_req_3; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_4: - s_n_llhttp__internal__n_start_req_4: { + case s_n_llhttp__internal__n_after_start_req_4: + s_n_llhttp__internal__n_after_start_req_4: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_4; + return s_n_llhttp__internal__n_after_start_req_4; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob20, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob18, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11011,23 +4540,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_4; + return s_n_llhttp__internal__n_after_start_req_4; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_6: - s_n_llhttp__internal__n_start_req_6: { + case s_n_llhttp__internal__n_after_start_req_6: + s_n_llhttp__internal__n_after_start_req_6: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_6; + return s_n_llhttp__internal__n_after_start_req_6; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob21, 6); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob19, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11036,23 +4564,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_6; + return s_n_llhttp__internal__n_after_start_req_6; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_8: - s_n_llhttp__internal__n_start_req_8: { + case s_n_llhttp__internal__n_after_start_req_8: + s_n_llhttp__internal__n_after_start_req_8: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_8; + return s_n_llhttp__internal__n_after_start_req_8; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob22, 4); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob20, 4); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11061,19 +4588,18 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_8; + return s_n_llhttp__internal__n_after_start_req_8; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_9: - s_n_llhttp__internal__n_start_req_9: { + case s_n_llhttp__internal__n_after_start_req_9: + s_n_llhttp__internal__n_after_start_req_9: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_9; + return s_n_llhttp__internal__n_after_start_req_9; } switch (*p) { case 'Y': { @@ -11082,62 +4608,59 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_7: - s_n_llhttp__internal__n_start_req_7: { + case s_n_llhttp__internal__n_after_start_req_7: + s_n_llhttp__internal__n_after_start_req_7: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_7; + return s_n_llhttp__internal__n_after_start_req_7; } switch (*p) { case 'N': { p++; - goto s_n_llhttp__internal__n_start_req_8; + goto s_n_llhttp__internal__n_after_start_req_8; } case 'P': { p++; - goto s_n_llhttp__internal__n_start_req_9; + goto s_n_llhttp__internal__n_after_start_req_9; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_5: - s_n_llhttp__internal__n_start_req_5: { + case s_n_llhttp__internal__n_after_start_req_5: + s_n_llhttp__internal__n_after_start_req_5: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_5; + return s_n_llhttp__internal__n_after_start_req_5; } switch (*p) { case 'H': { p++; - goto s_n_llhttp__internal__n_start_req_6; + goto s_n_llhttp__internal__n_after_start_req_6; } case 'O': { p++; - goto s_n_llhttp__internal__n_start_req_7; + goto s_n_llhttp__internal__n_after_start_req_7; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_12: - s_n_llhttp__internal__n_start_req_12: { + case s_n_llhttp__internal__n_after_start_req_12: + s_n_llhttp__internal__n_after_start_req_12: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_12; + return s_n_llhttp__internal__n_after_start_req_12; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob23, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob21, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11146,23 +4669,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_12; + return s_n_llhttp__internal__n_after_start_req_12; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_13: - s_n_llhttp__internal__n_start_req_13: { + case s_n_llhttp__internal__n_after_start_req_13: + s_n_llhttp__internal__n_after_start_req_13: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_13; + return s_n_llhttp__internal__n_after_start_req_13; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob24, 5); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob22, 5); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11171,61 +4693,58 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_13; + return s_n_llhttp__internal__n_after_start_req_13; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_11: - s_n_llhttp__internal__n_start_req_11: { + case s_n_llhttp__internal__n_after_start_req_11: + s_n_llhttp__internal__n_after_start_req_11: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_11; + return s_n_llhttp__internal__n_after_start_req_11; } switch (*p) { case 'L': { p++; - goto s_n_llhttp__internal__n_start_req_12; + goto s_n_llhttp__internal__n_after_start_req_12; } case 'S': { p++; - goto s_n_llhttp__internal__n_start_req_13; + goto s_n_llhttp__internal__n_after_start_req_13; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_10: - s_n_llhttp__internal__n_start_req_10: { + case s_n_llhttp__internal__n_after_start_req_10: + s_n_llhttp__internal__n_after_start_req_10: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_10; + return s_n_llhttp__internal__n_after_start_req_10; } switch (*p) { case 'E': { p++; - goto s_n_llhttp__internal__n_start_req_11; + goto s_n_llhttp__internal__n_after_start_req_11; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_14: - s_n_llhttp__internal__n_start_req_14: { + case s_n_llhttp__internal__n_after_start_req_14: + s_n_llhttp__internal__n_after_start_req_14: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_14; + return s_n_llhttp__internal__n_after_start_req_14; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob25, 4); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob23, 4); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11234,23 +4753,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_14; + return s_n_llhttp__internal__n_after_start_req_14; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_17: - s_n_llhttp__internal__n_start_req_17: { + case s_n_llhttp__internal__n_after_start_req_17: + s_n_llhttp__internal__n_after_start_req_17: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_17; + return s_n_llhttp__internal__n_after_start_req_17; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob27, 9); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob25, 9); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11259,65 +4777,62 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_17; + return s_n_llhttp__internal__n_after_start_req_17; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_16: - s_n_llhttp__internal__n_start_req_16: { + case s_n_llhttp__internal__n_after_start_req_16: + s_n_llhttp__internal__n_after_start_req_16: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_16; + return s_n_llhttp__internal__n_after_start_req_16; } switch (*p) { case '_': { p++; - goto s_n_llhttp__internal__n_start_req_17; + goto s_n_llhttp__internal__n_after_start_req_17; } default: { match = 1; goto s_n_llhttp__internal__n_invoke_store_method_1; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_15: - s_n_llhttp__internal__n_start_req_15: { + case s_n_llhttp__internal__n_after_start_req_15: + s_n_llhttp__internal__n_after_start_req_15: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_15; + return s_n_llhttp__internal__n_after_start_req_15; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob26, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob24, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { p++; - goto s_n_llhttp__internal__n_start_req_16; + goto s_n_llhttp__internal__n_after_start_req_16; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_15; + return s_n_llhttp__internal__n_after_start_req_15; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_18: - s_n_llhttp__internal__n_start_req_18: { + case s_n_llhttp__internal__n_after_start_req_18: + s_n_llhttp__internal__n_after_start_req_18: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_18; + return s_n_llhttp__internal__n_after_start_req_18; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob28, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob26, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11326,23 +4841,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_18; + return s_n_llhttp__internal__n_after_start_req_18; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_20: - s_n_llhttp__internal__n_start_req_20: { + case s_n_llhttp__internal__n_after_start_req_20: + s_n_llhttp__internal__n_after_start_req_20: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_20; + return s_n_llhttp__internal__n_after_start_req_20; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob29, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob27, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11351,23 +4865,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_20; + return s_n_llhttp__internal__n_after_start_req_20; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_21: - s_n_llhttp__internal__n_start_req_21: { + case s_n_llhttp__internal__n_after_start_req_21: + s_n_llhttp__internal__n_after_start_req_21: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_21; + return s_n_llhttp__internal__n_after_start_req_21; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob30, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob28, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11376,44 +4889,42 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_21; + return s_n_llhttp__internal__n_after_start_req_21; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_19: - s_n_llhttp__internal__n_start_req_19: { + case s_n_llhttp__internal__n_after_start_req_19: + s_n_llhttp__internal__n_after_start_req_19: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_19; + return s_n_llhttp__internal__n_after_start_req_19; } switch (*p) { case 'I': { p++; - goto s_n_llhttp__internal__n_start_req_20; + goto s_n_llhttp__internal__n_after_start_req_20; } case 'O': { p++; - goto s_n_llhttp__internal__n_start_req_21; + goto s_n_llhttp__internal__n_after_start_req_21; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_23: - s_n_llhttp__internal__n_start_req_23: { + case s_n_llhttp__internal__n_after_start_req_23: + s_n_llhttp__internal__n_after_start_req_23: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_23; + return s_n_llhttp__internal__n_after_start_req_23; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob31, 6); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob29, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11422,23 +4933,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_23; + return s_n_llhttp__internal__n_after_start_req_23; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_24: - s_n_llhttp__internal__n_start_req_24: { + case s_n_llhttp__internal__n_after_start_req_24: + s_n_llhttp__internal__n_after_start_req_24: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_24; + return s_n_llhttp__internal__n_after_start_req_24; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob32, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob30, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11447,23 +4957,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_24; + return s_n_llhttp__internal__n_after_start_req_24; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_26: - s_n_llhttp__internal__n_start_req_26: { + case s_n_llhttp__internal__n_after_start_req_26: + s_n_llhttp__internal__n_after_start_req_26: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_26; + return s_n_llhttp__internal__n_after_start_req_26; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob33, 7); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob31, 7); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11472,23 +4981,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_26; + return s_n_llhttp__internal__n_after_start_req_26; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_28: - s_n_llhttp__internal__n_start_req_28: { + case s_n_llhttp__internal__n_after_start_req_28: + s_n_llhttp__internal__n_after_start_req_28: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_28; + return s_n_llhttp__internal__n_after_start_req_28; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob34, 6); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob32, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11497,19 +5005,18 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_28; + return s_n_llhttp__internal__n_after_start_req_28; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_29: - s_n_llhttp__internal__n_start_req_29: { + case s_n_llhttp__internal__n_after_start_req_29: + s_n_llhttp__internal__n_after_start_req_29: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_29; + return s_n_llhttp__internal__n_after_start_req_29; } switch (*p) { case 'L': { @@ -11518,62 +5025,59 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_27: - s_n_llhttp__internal__n_start_req_27: { + case s_n_llhttp__internal__n_after_start_req_27: + s_n_llhttp__internal__n_after_start_req_27: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_27; + return s_n_llhttp__internal__n_after_start_req_27; } switch (*p) { case 'A': { p++; - goto s_n_llhttp__internal__n_start_req_28; + goto s_n_llhttp__internal__n_after_start_req_28; } case 'O': { p++; - goto s_n_llhttp__internal__n_start_req_29; + goto s_n_llhttp__internal__n_after_start_req_29; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_25: - s_n_llhttp__internal__n_start_req_25: { + case s_n_llhttp__internal__n_after_start_req_25: + s_n_llhttp__internal__n_after_start_req_25: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_25; + return s_n_llhttp__internal__n_after_start_req_25; } switch (*p) { case 'A': { p++; - goto s_n_llhttp__internal__n_start_req_26; + goto s_n_llhttp__internal__n_after_start_req_26; } case 'C': { p++; - goto s_n_llhttp__internal__n_start_req_27; + goto s_n_llhttp__internal__n_after_start_req_27; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_30: - s_n_llhttp__internal__n_start_req_30: { + case s_n_llhttp__internal__n_after_start_req_30: + s_n_llhttp__internal__n_after_start_req_30: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_30; + return s_n_llhttp__internal__n_after_start_req_30; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob35, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob33, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11582,52 +5086,50 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_30; + return s_n_llhttp__internal__n_after_start_req_30; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_22: - s_n_llhttp__internal__n_start_req_22: { + case s_n_llhttp__internal__n_after_start_req_22: + s_n_llhttp__internal__n_after_start_req_22: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_22; + return s_n_llhttp__internal__n_after_start_req_22; } switch (*p) { case '-': { p++; - goto s_n_llhttp__internal__n_start_req_23; + goto s_n_llhttp__internal__n_after_start_req_23; } case 'E': { p++; - goto s_n_llhttp__internal__n_start_req_24; + goto s_n_llhttp__internal__n_after_start_req_24; } case 'K': { p++; - goto s_n_llhttp__internal__n_start_req_25; + goto s_n_llhttp__internal__n_after_start_req_25; } case 'O': { p++; - goto s_n_llhttp__internal__n_start_req_30; + goto s_n_llhttp__internal__n_after_start_req_30; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_31: - s_n_llhttp__internal__n_start_req_31: { + case s_n_llhttp__internal__n_after_start_req_31: + s_n_llhttp__internal__n_after_start_req_31: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_31; + return s_n_llhttp__internal__n_after_start_req_31; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob36, 5); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob34, 5); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11636,23 +5138,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_31; + return s_n_llhttp__internal__n_after_start_req_31; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_32: - s_n_llhttp__internal__n_start_req_32: { + case s_n_llhttp__internal__n_after_start_req_32: + s_n_llhttp__internal__n_after_start_req_32: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_32; + return s_n_llhttp__internal__n_after_start_req_32; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob37, 6); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob35, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11661,23 +5162,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_32; + return s_n_llhttp__internal__n_after_start_req_32; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_35: - s_n_llhttp__internal__n_start_req_35: { + case s_n_llhttp__internal__n_after_start_req_35: + s_n_llhttp__internal__n_after_start_req_35: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_35; + return s_n_llhttp__internal__n_after_start_req_35; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob38, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob36, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11686,23 +5186,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_35; + return s_n_llhttp__internal__n_after_start_req_35; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_36: - s_n_llhttp__internal__n_start_req_36: { + case s_n_llhttp__internal__n_after_start_req_36: + s_n_llhttp__internal__n_after_start_req_36: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_36; + return s_n_llhttp__internal__n_after_start_req_36; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob39, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob37, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11711,44 +5210,42 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_36; + return s_n_llhttp__internal__n_after_start_req_36; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_34: - s_n_llhttp__internal__n_start_req_34: { + case s_n_llhttp__internal__n_after_start_req_34: + s_n_llhttp__internal__n_after_start_req_34: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_34; + return s_n_llhttp__internal__n_after_start_req_34; } switch (*p) { case 'T': { p++; - goto s_n_llhttp__internal__n_start_req_35; + goto s_n_llhttp__internal__n_after_start_req_35; } case 'U': { p++; - goto s_n_llhttp__internal__n_start_req_36; + goto s_n_llhttp__internal__n_after_start_req_36; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_37: - s_n_llhttp__internal__n_start_req_37: { + case s_n_llhttp__internal__n_after_start_req_37: + s_n_llhttp__internal__n_after_start_req_37: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_37; + return s_n_llhttp__internal__n_after_start_req_37; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob40, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob38, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11757,23 +5254,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_37; + return s_n_llhttp__internal__n_after_start_req_37; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_38: - s_n_llhttp__internal__n_start_req_38: { + case s_n_llhttp__internal__n_after_start_req_38: + s_n_llhttp__internal__n_after_start_req_38: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_38; + return s_n_llhttp__internal__n_after_start_req_38; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob41, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob39, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11782,23 +5278,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_38; + return s_n_llhttp__internal__n_after_start_req_38; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_42: - s_n_llhttp__internal__n_start_req_42: { + case s_n_llhttp__internal__n_after_start_req_42: + s_n_llhttp__internal__n_after_start_req_42: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_42; + return s_n_llhttp__internal__n_after_start_req_42; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob42, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob40, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11807,23 +5302,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_42; + return s_n_llhttp__internal__n_after_start_req_42; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_43: - s_n_llhttp__internal__n_start_req_43: { + case s_n_llhttp__internal__n_after_start_req_43: + s_n_llhttp__internal__n_after_start_req_43: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_43; + return s_n_llhttp__internal__n_after_start_req_43; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob43, 4); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob41, 4); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11832,57 +5326,54 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_43; + return s_n_llhttp__internal__n_after_start_req_43; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_41: - s_n_llhttp__internal__n_start_req_41: { + case s_n_llhttp__internal__n_after_start_req_41: + s_n_llhttp__internal__n_after_start_req_41: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_41; + return s_n_llhttp__internal__n_after_start_req_41; } switch (*p) { case 'F': { p++; - goto s_n_llhttp__internal__n_start_req_42; + goto s_n_llhttp__internal__n_after_start_req_42; } case 'P': { p++; - goto s_n_llhttp__internal__n_start_req_43; + goto s_n_llhttp__internal__n_after_start_req_43; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_40: - s_n_llhttp__internal__n_start_req_40: { + case s_n_llhttp__internal__n_after_start_req_40: + s_n_llhttp__internal__n_after_start_req_40: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_40; + return s_n_llhttp__internal__n_after_start_req_40; } switch (*p) { case 'P': { p++; - goto s_n_llhttp__internal__n_start_req_41; + goto s_n_llhttp__internal__n_after_start_req_41; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_39: - s_n_llhttp__internal__n_start_req_39: { + case s_n_llhttp__internal__n_after_start_req_39: + s_n_llhttp__internal__n_after_start_req_39: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_39; + return s_n_llhttp__internal__n_after_start_req_39; } switch (*p) { case 'I': { @@ -11892,23 +5383,22 @@ static llparse_state_t llhttp__internal__run( } case 'O': { p++; - goto s_n_llhttp__internal__n_start_req_40; + goto s_n_llhttp__internal__n_after_start_req_40; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_45: - s_n_llhttp__internal__n_start_req_45: { + case s_n_llhttp__internal__n_after_start_req_45: + s_n_llhttp__internal__n_after_start_req_45: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_45; + return s_n_llhttp__internal__n_after_start_req_45; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob44, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob42, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11917,24 +5407,23 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_45; + return s_n_llhttp__internal__n_after_start_req_45; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_44: - s_n_llhttp__internal__n_start_req_44: { + case s_n_llhttp__internal__n_after_start_req_44: + s_n_llhttp__internal__n_after_start_req_44: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_44; + return s_n_llhttp__internal__n_after_start_req_44; } switch (*p) { case 'R': { p++; - goto s_n_llhttp__internal__n_start_req_45; + goto s_n_llhttp__internal__n_after_start_req_45; } case 'T': { p++; @@ -11942,53 +5431,75 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_33: - s_n_llhttp__internal__n_start_req_33: { + case s_n_llhttp__internal__n_after_start_req_33: + s_n_llhttp__internal__n_after_start_req_33: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_33; + return s_n_llhttp__internal__n_after_start_req_33; } switch (*p) { case 'A': { p++; - goto s_n_llhttp__internal__n_start_req_34; + goto s_n_llhttp__internal__n_after_start_req_34; } case 'L': { p++; - goto s_n_llhttp__internal__n_start_req_37; + goto s_n_llhttp__internal__n_after_start_req_37; } case 'O': { p++; - goto s_n_llhttp__internal__n_start_req_38; + goto s_n_llhttp__internal__n_after_start_req_38; } case 'R': { p++; - goto s_n_llhttp__internal__n_start_req_39; + goto s_n_llhttp__internal__n_after_start_req_39; } case 'U': { p++; - goto s_n_llhttp__internal__n_start_req_44; + goto s_n_llhttp__internal__n_after_start_req_44; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_48: - s_n_llhttp__internal__n_start_req_48: { + case s_n_llhttp__internal__n_after_start_req_46: + s_n_llhttp__internal__n_after_start_req_46: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_48; + return s_n_llhttp__internal__n_after_start_req_46; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob45, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob43, 4); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + match = 46; + goto s_n_llhttp__internal__n_invoke_store_method_1; + } + case kMatchPause: { + return s_n_llhttp__internal__n_after_start_req_46; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_error_112; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_after_start_req_49: + s_n_llhttp__internal__n_after_start_req_49: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_after_start_req_49; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob44, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -11997,23 +5508,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_48; + return s_n_llhttp__internal__n_after_start_req_49; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_49: - s_n_llhttp__internal__n_start_req_49: { + case s_n_llhttp__internal__n_after_start_req_50: + s_n_llhttp__internal__n_after_start_req_50: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_49; + return s_n_llhttp__internal__n_after_start_req_50; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob46, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob45, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12022,23 +5532,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_49; + return s_n_llhttp__internal__n_after_start_req_50; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_50: - s_n_llhttp__internal__n_start_req_50: { + case s_n_llhttp__internal__n_after_start_req_51: + s_n_llhttp__internal__n_after_start_req_51: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_50; + return s_n_llhttp__internal__n_after_start_req_51; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob47, 5); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob46, 5); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12047,23 +5556,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_50; + return s_n_llhttp__internal__n_after_start_req_51; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_51: - s_n_llhttp__internal__n_start_req_51: { + case s_n_llhttp__internal__n_after_start_req_52: + s_n_llhttp__internal__n_after_start_req_52: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_51; + return s_n_llhttp__internal__n_after_start_req_52; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob48, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob47, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12072,69 +5580,66 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_51; + return s_n_llhttp__internal__n_after_start_req_52; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_47: - s_n_llhttp__internal__n_start_req_47: { + case s_n_llhttp__internal__n_after_start_req_48: + s_n_llhttp__internal__n_after_start_req_48: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_47; + return s_n_llhttp__internal__n_after_start_req_48; } switch (*p) { case 'B': { p++; - goto s_n_llhttp__internal__n_start_req_48; + goto s_n_llhttp__internal__n_after_start_req_49; } case 'C': { p++; - goto s_n_llhttp__internal__n_start_req_49; + goto s_n_llhttp__internal__n_after_start_req_50; } case 'D': { p++; - goto s_n_llhttp__internal__n_start_req_50; + goto s_n_llhttp__internal__n_after_start_req_51; } case 'P': { p++; - goto s_n_llhttp__internal__n_start_req_51; + goto s_n_llhttp__internal__n_after_start_req_52; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_46: - s_n_llhttp__internal__n_start_req_46: { + case s_n_llhttp__internal__n_after_start_req_47: + s_n_llhttp__internal__n_after_start_req_47: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_46; + return s_n_llhttp__internal__n_after_start_req_47; } switch (*p) { case 'E': { p++; - goto s_n_llhttp__internal__n_start_req_47; + goto s_n_llhttp__internal__n_after_start_req_48; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_54: - s_n_llhttp__internal__n_start_req_54: { + case s_n_llhttp__internal__n_after_start_req_55: + s_n_llhttp__internal__n_after_start_req_55: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_54; + return s_n_llhttp__internal__n_after_start_req_55; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob49, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob48, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12143,19 +5648,18 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_54; + return s_n_llhttp__internal__n_after_start_req_55; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_56: - s_n_llhttp__internal__n_start_req_56: { + case s_n_llhttp__internal__n_after_start_req_57: + s_n_llhttp__internal__n_after_start_req_57: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_56; + return s_n_llhttp__internal__n_after_start_req_57; } switch (*p) { case 'P': { @@ -12164,20 +5668,19 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_57: - s_n_llhttp__internal__n_start_req_57: { + case s_n_llhttp__internal__n_after_start_req_58: + s_n_llhttp__internal__n_after_start_req_58: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_57; + return s_n_llhttp__internal__n_after_start_req_58; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob50, 9); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob49, 9); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12186,65 +5689,62 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_57; + return s_n_llhttp__internal__n_after_start_req_58; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_55: - s_n_llhttp__internal__n_start_req_55: { + case s_n_llhttp__internal__n_after_start_req_56: + s_n_llhttp__internal__n_after_start_req_56: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_55; + return s_n_llhttp__internal__n_after_start_req_56; } switch (*p) { case 'U': { p++; - goto s_n_llhttp__internal__n_start_req_56; + goto s_n_llhttp__internal__n_after_start_req_57; } case '_': { p++; - goto s_n_llhttp__internal__n_start_req_57; + goto s_n_llhttp__internal__n_after_start_req_58; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_53: - s_n_llhttp__internal__n_start_req_53: { + case s_n_llhttp__internal__n_after_start_req_54: + s_n_llhttp__internal__n_after_start_req_54: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_53; + return s_n_llhttp__internal__n_after_start_req_54; } switch (*p) { case 'A': { p++; - goto s_n_llhttp__internal__n_start_req_54; + goto s_n_llhttp__internal__n_after_start_req_55; } case 'T': { p++; - goto s_n_llhttp__internal__n_start_req_55; + goto s_n_llhttp__internal__n_after_start_req_56; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_58: - s_n_llhttp__internal__n_start_req_58: { + case s_n_llhttp__internal__n_after_start_req_59: + s_n_llhttp__internal__n_after_start_req_59: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_58; + return s_n_llhttp__internal__n_after_start_req_59; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob51, 4); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob50, 4); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12253,23 +5753,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_58; + return s_n_llhttp__internal__n_after_start_req_59; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_59: - s_n_llhttp__internal__n_start_req_59: { + case s_n_llhttp__internal__n_after_start_req_60: + s_n_llhttp__internal__n_after_start_req_60: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_59; + return s_n_llhttp__internal__n_after_start_req_60; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob52, 7); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob51, 7); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12278,48 +5777,46 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_59; + return s_n_llhttp__internal__n_after_start_req_60; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_52: - s_n_llhttp__internal__n_start_req_52: { + case s_n_llhttp__internal__n_after_start_req_53: + s_n_llhttp__internal__n_after_start_req_53: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_52; + return s_n_llhttp__internal__n_after_start_req_53; } switch (*p) { case 'E': { p++; - goto s_n_llhttp__internal__n_start_req_53; + goto s_n_llhttp__internal__n_after_start_req_54; } case 'O': { p++; - goto s_n_llhttp__internal__n_start_req_58; + goto s_n_llhttp__internal__n_after_start_req_59; } case 'U': { p++; - goto s_n_llhttp__internal__n_start_req_59; + goto s_n_llhttp__internal__n_after_start_req_60; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_61: - s_n_llhttp__internal__n_start_req_61: { + case s_n_llhttp__internal__n_after_start_req_62: + s_n_llhttp__internal__n_after_start_req_62: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_61; + return s_n_llhttp__internal__n_after_start_req_62; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob53, 6); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob52, 6); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12328,23 +5825,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_61; + return s_n_llhttp__internal__n_after_start_req_62; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_62: - s_n_llhttp__internal__n_start_req_62: { + case s_n_llhttp__internal__n_after_start_req_63: + s_n_llhttp__internal__n_after_start_req_63: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_62; + return s_n_llhttp__internal__n_after_start_req_63; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob54, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob53, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12353,44 +5849,42 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_62; + return s_n_llhttp__internal__n_after_start_req_63; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_60: - s_n_llhttp__internal__n_start_req_60: { + case s_n_llhttp__internal__n_after_start_req_61: + s_n_llhttp__internal__n_after_start_req_61: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_60; + return s_n_llhttp__internal__n_after_start_req_61; } switch (*p) { case 'E': { p++; - goto s_n_llhttp__internal__n_start_req_61; + goto s_n_llhttp__internal__n_after_start_req_62; } case 'R': { p++; - goto s_n_llhttp__internal__n_start_req_62; + goto s_n_llhttp__internal__n_after_start_req_63; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_65: - s_n_llhttp__internal__n_start_req_65: { + case s_n_llhttp__internal__n_after_start_req_66: + s_n_llhttp__internal__n_after_start_req_66: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_65; + return s_n_llhttp__internal__n_after_start_req_66; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob55, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob54, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12399,23 +5893,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_65; + return s_n_llhttp__internal__n_after_start_req_66; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_67: - s_n_llhttp__internal__n_start_req_67: { + case s_n_llhttp__internal__n_after_start_req_68: + s_n_llhttp__internal__n_after_start_req_68: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_67; + return s_n_llhttp__internal__n_after_start_req_68; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob56, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob55, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12424,23 +5917,22 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_67; + return s_n_llhttp__internal__n_after_start_req_68; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_68: - s_n_llhttp__internal__n_start_req_68: { + case s_n_llhttp__internal__n_after_start_req_69: + s_n_llhttp__internal__n_after_start_req_69: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_68; + return s_n_llhttp__internal__n_after_start_req_69; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob57, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob56, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12449,44 +5941,42 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_68; + return s_n_llhttp__internal__n_after_start_req_69; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_66: - s_n_llhttp__internal__n_start_req_66: { + case s_n_llhttp__internal__n_after_start_req_67: + s_n_llhttp__internal__n_after_start_req_67: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_66; + return s_n_llhttp__internal__n_after_start_req_67; } switch (*p) { case 'I': { p++; - goto s_n_llhttp__internal__n_start_req_67; + goto s_n_llhttp__internal__n_after_start_req_68; } case 'O': { p++; - goto s_n_llhttp__internal__n_start_req_68; + goto s_n_llhttp__internal__n_after_start_req_69; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_69: - s_n_llhttp__internal__n_start_req_69: { + case s_n_llhttp__internal__n_after_start_req_70: + s_n_llhttp__internal__n_after_start_req_70: { llparse_match_t match_seq; - + if (p == endp) { - return s_n_llhttp__internal__n_start_req_69; + return s_n_llhttp__internal__n_after_start_req_70; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob58, 8); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob57, 8); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12495,152 +5985,173 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_method_1; } case kMatchPause: { - return s_n_llhttp__internal__n_start_req_69; + return s_n_llhttp__internal__n_after_start_req_70; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_64: - s_n_llhttp__internal__n_start_req_64: { + case s_n_llhttp__internal__n_after_start_req_65: + s_n_llhttp__internal__n_after_start_req_65: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_64; + return s_n_llhttp__internal__n_after_start_req_65; } switch (*p) { case 'B': { p++; - goto s_n_llhttp__internal__n_start_req_65; + goto s_n_llhttp__internal__n_after_start_req_66; } case 'L': { p++; - goto s_n_llhttp__internal__n_start_req_66; + goto s_n_llhttp__internal__n_after_start_req_67; } case 'S': { p++; - goto s_n_llhttp__internal__n_start_req_69; + goto s_n_llhttp__internal__n_after_start_req_70; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req_63: - s_n_llhttp__internal__n_start_req_63: { + case s_n_llhttp__internal__n_after_start_req_64: + s_n_llhttp__internal__n_after_start_req_64: { if (p == endp) { - return s_n_llhttp__internal__n_start_req_63; + return s_n_llhttp__internal__n_after_start_req_64; } switch (*p) { case 'N': { p++; - goto s_n_llhttp__internal__n_start_req_64; + goto s_n_llhttp__internal__n_after_start_req_65; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_req: - s_n_llhttp__internal__n_start_req: { + case s_n_llhttp__internal__n_after_start_req: + s_n_llhttp__internal__n_after_start_req: { if (p == endp) { - return s_n_llhttp__internal__n_start_req; + return s_n_llhttp__internal__n_after_start_req; } switch (*p) { case 'A': { p++; - goto s_n_llhttp__internal__n_start_req_1; + goto s_n_llhttp__internal__n_after_start_req_1; } case 'B': { p++; - goto s_n_llhttp__internal__n_start_req_4; + goto s_n_llhttp__internal__n_after_start_req_4; } case 'C': { p++; - goto s_n_llhttp__internal__n_start_req_5; + goto s_n_llhttp__internal__n_after_start_req_5; } case 'D': { p++; - goto s_n_llhttp__internal__n_start_req_10; + goto s_n_llhttp__internal__n_after_start_req_10; } case 'F': { p++; - goto s_n_llhttp__internal__n_start_req_14; + goto s_n_llhttp__internal__n_after_start_req_14; } case 'G': { p++; - goto s_n_llhttp__internal__n_start_req_15; + goto s_n_llhttp__internal__n_after_start_req_15; } case 'H': { p++; - goto s_n_llhttp__internal__n_start_req_18; + goto s_n_llhttp__internal__n_after_start_req_18; } case 'L': { p++; - goto s_n_llhttp__internal__n_start_req_19; + goto s_n_llhttp__internal__n_after_start_req_19; } case 'M': { p++; - goto s_n_llhttp__internal__n_start_req_22; + goto s_n_llhttp__internal__n_after_start_req_22; } case 'N': { p++; - goto s_n_llhttp__internal__n_start_req_31; + goto s_n_llhttp__internal__n_after_start_req_31; } case 'O': { p++; - goto s_n_llhttp__internal__n_start_req_32; + goto s_n_llhttp__internal__n_after_start_req_32; } case 'P': { p++; - goto s_n_llhttp__internal__n_start_req_33; + goto s_n_llhttp__internal__n_after_start_req_33; + } + case 'Q': { + p++; + goto s_n_llhttp__internal__n_after_start_req_46; } case 'R': { p++; - goto s_n_llhttp__internal__n_start_req_46; + goto s_n_llhttp__internal__n_after_start_req_47; } case 'S': { p++; - goto s_n_llhttp__internal__n_start_req_52; + goto s_n_llhttp__internal__n_after_start_req_53; } case 'T': { p++; - goto s_n_llhttp__internal__n_start_req_60; + goto s_n_llhttp__internal__n_after_start_req_61; } case 'U': { p++; - goto s_n_llhttp__internal__n_start_req_63; + goto s_n_llhttp__internal__n_after_start_req_64; } default: { - goto s_n_llhttp__internal__n_error_49; + goto s_n_llhttp__internal__n_error_112; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_invoke_llhttp__on_status_complete: - s_n_llhttp__internal__n_invoke_llhttp__on_status_complete: { - switch (llhttp__on_status_complete(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_field_start; + case s_n_llhttp__internal__n_span_start_llhttp__on_method_1: + s_n_llhttp__internal__n_span_start_llhttp__on_method_1: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_method_1; } - /* UNREACHABLE */; - abort(); + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_method; + goto s_n_llhttp__internal__n_after_start_req; + UNREACHABLE; } case s_n_llhttp__internal__n_res_line_almost_done: s_n_llhttp__internal__n_res_line_almost_done: { if (p == endp) { return s_n_llhttp__internal__n_res_line_almost_done; } - p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; - /* UNREACHABLE */; - abort(); + switch (*p) { + case 10: { + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + } + case 13: { + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + } + default: { + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_29; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_test_lenient_flags_30: + s_n_llhttp__internal__n_invoke_test_lenient_flags_30: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + default: + goto s_n_llhttp__internal__n_error_98; + } + UNREACHABLE; } case s_n_llhttp__internal__n_res_status: s_n_llhttp__internal__n_res_status: { @@ -12659,8 +6170,7 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_res_status; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_span_start_llhttp__on_status: s_n_llhttp__internal__n_span_start_llhttp__on_status: { @@ -12670,29 +6180,7 @@ static llparse_state_t llhttp__internal__run( state->_span_pos0 = (void*) p; state->_span_cb0 = llhttp__on_status; goto s_n_llhttp__internal__n_res_status; - /* UNREACHABLE */; - abort(); - } - case s_n_llhttp__internal__n_res_status_start: - s_n_llhttp__internal__n_res_status_start: { - if (p == endp) { - return s_n_llhttp__internal__n_res_status_start; - } - switch (*p) { - case 10: { - p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; - } - case 13: { - p++; - goto s_n_llhttp__internal__n_res_line_almost_done; - } - default: { - goto s_n_llhttp__internal__n_span_start_llhttp__on_status; - } - } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_res_status_code_otherwise: s_n_llhttp__internal__n_res_status_code_otherwise: { @@ -12701,26 +6189,151 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case 10: { - goto s_n_llhttp__internal__n_res_status_start; + p++; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_28; } case 13: { - goto s_n_llhttp__internal__n_res_status_start; + p++; + goto s_n_llhttp__internal__n_res_line_almost_done; } case ' ': { p++; - goto s_n_llhttp__internal__n_res_status_start; + goto s_n_llhttp__internal__n_span_start_llhttp__on_status; } default: { - goto s_n_llhttp__internal__n_error_43; + goto s_n_llhttp__internal__n_error_99; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_res_status_code: - s_n_llhttp__internal__n_res_status_code: { + case s_n_llhttp__internal__n_res_status_code_digit_3: + s_n_llhttp__internal__n_res_status_code_digit_3: { if (p == endp) { - return s_n_llhttp__internal__n_res_status_code; + return s_n_llhttp__internal__n_res_status_code_digit_3; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_2; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_2; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_2; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_2; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_2; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_2; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_2; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_2; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_2; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_2; + } + default: { + goto s_n_llhttp__internal__n_error_101; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_res_status_code_digit_2: + s_n_llhttp__internal__n_res_status_code_digit_2: { + if (p == endp) { + return s_n_llhttp__internal__n_res_status_code_digit_2; + } + switch (*p) { + case '0': { + p++; + match = 0; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_1; + } + case '1': { + p++; + match = 1; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_1; + } + case '2': { + p++; + match = 2; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_1; + } + case '3': { + p++; + match = 3; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_1; + } + case '4': { + p++; + match = 4; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_1; + } + case '5': { + p++; + match = 5; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_1; + } + case '6': { + p++; + match = 6; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_1; + } + case '7': { + p++; + match = 7; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_1; + } + case '8': { + p++; + match = 8; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_1; + } + case '9': { + p++; + match = 9; + goto s_n_llhttp__internal__n_invoke_mul_add_status_code_1; + } + default: { + goto s_n_llhttp__internal__n_error_103; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_res_status_code_digit_1: + s_n_llhttp__internal__n_res_status_code_digit_1: { + if (p == endp) { + return s_n_llhttp__internal__n_res_status_code_digit_1; } switch (*p) { case '0': { @@ -12774,16 +6387,15 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_mul_add_status_code; } default: { - goto s_n_llhttp__internal__n_res_status_code_otherwise; + goto s_n_llhttp__internal__n_error_105; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_res_http_end: - s_n_llhttp__internal__n_res_http_end: { + case s_n_llhttp__internal__n_res_after_version: + s_n_llhttp__internal__n_res_after_version: { if (p == endp) { - return s_n_llhttp__internal__n_res_http_end; + return s_n_llhttp__internal__n_res_after_version; } switch (*p) { case ' ': { @@ -12791,11 +6403,40 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_update_status_code; } default: { - goto s_n_llhttp__internal__n_error_44; + goto s_n_llhttp__internal__n_error_106; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_version_complete_1: + s_n_llhttp__internal__n_invoke_llhttp__on_version_complete_1: { + switch (llhttp__on_version_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_res_after_version; + case 21: + goto s_n_llhttp__internal__n_pause_28; + default: + goto s_n_llhttp__internal__n_error_94; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_93: + s_n_llhttp__internal__n_error_93: { + state->error = 0x9; + state->reason = "Invalid HTTP version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_107: + s_n_llhttp__internal__n_error_107: { + state->error = 0x9; + state->reason = "Invalid minor version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } case s_n_llhttp__internal__n_res_http_minor: s_n_llhttp__internal__n_res_http_minor: { @@ -12854,11 +6495,19 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_http_minor_1; } default: { - goto s_n_llhttp__internal__n_error_45; + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_7; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_108: + s_n_llhttp__internal__n_error_108: { + state->error = 0x9; + state->reason = "Expected dot"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } case s_n_llhttp__internal__n_res_http_dot: s_n_llhttp__internal__n_res_http_dot: { @@ -12871,11 +6520,19 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_res_http_minor; } default: { - goto s_n_llhttp__internal__n_error_46; + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_8; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_109: + s_n_llhttp__internal__n_error_109: { + state->error = 0x9; + state->reason = "Invalid major version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } case s_n_llhttp__internal__n_res_http_major: s_n_llhttp__internal__n_res_http_major: { @@ -12934,44 +6591,181 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_invoke_store_http_major_1; } default: { - goto s_n_llhttp__internal__n_error_47; + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_9; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - case s_n_llhttp__internal__n_start_res: - s_n_llhttp__internal__n_start_res: { - llparse_match_t match_seq; - + case s_n_llhttp__internal__n_span_start_llhttp__on_version_1: + s_n_llhttp__internal__n_span_start_llhttp__on_version_1: { if (p == endp) { - return s_n_llhttp__internal__n_start_res; + return s_n_llhttp__internal__n_span_start_llhttp__on_version_1; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob59, 5); + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_version; + goto s_n_llhttp__internal__n_res_http_major; + UNREACHABLE; + } + case s_n_llhttp__internal__n_res_after_protocol: + s_n_llhttp__internal__n_res_after_protocol: { + if (p == endp) { + return s_n_llhttp__internal__n_res_after_protocol; + } + switch (*p) { + case '/': { + p++; + goto s_n_llhttp__internal__n_span_start_llhttp__on_version_1; + } + default: { + goto s_n_llhttp__internal__n_error_114; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_3: + s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_3: { + switch (llhttp__on_protocol_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_res_after_protocol; + case 21: + goto s_n_llhttp__internal__n_pause_30; + default: + goto s_n_llhttp__internal__n_error_113; + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_error_115: + s_n_llhttp__internal__n_error_115: { + state->error = 0x8; + state->reason = "Expected HTTP/, RTSP/ or ICE/"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + case s_n_llhttp__internal__n_res_after_start_1: + s_n_llhttp__internal__n_res_after_start_1: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_res_after_start_1; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob58, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { p++; - goto s_n_llhttp__internal__n_res_http_major; + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_4; } case kMatchPause: { - return s_n_llhttp__internal__n_start_res; + return s_n_llhttp__internal__n_res_after_start_1; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_50; + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_5; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_res_after_start_2: + s_n_llhttp__internal__n_res_after_start_2: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_res_after_start_2; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob59, 2); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_4; + } + case kMatchPause: { + return s_n_llhttp__internal__n_res_after_start_2; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_5; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_res_after_start_3: + s_n_llhttp__internal__n_res_after_start_3: { + llparse_match_t match_seq; + + if (p == endp) { + return s_n_llhttp__internal__n_res_after_start_3; + } + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob60, 3); + p = match_seq.current; + switch (match_seq.status) { + case kMatchComplete: { + p++; + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_4; + } + case kMatchPause: { + return s_n_llhttp__internal__n_res_after_start_3; + } + case kMatchMismatch: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_5; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_res_after_start: + s_n_llhttp__internal__n_res_after_start: { + if (p == endp) { + return s_n_llhttp__internal__n_res_after_start; + } + switch (*p) { + case 'H': { + p++; + goto s_n_llhttp__internal__n_res_after_start_1; + } + case 'I': { + p++; + goto s_n_llhttp__internal__n_res_after_start_2; + } + case 'R': { + p++; + goto s_n_llhttp__internal__n_res_after_start_3; + } + default: { + goto s_n_llhttp__internal__n_span_end_llhttp__on_protocol_5; + } + } + UNREACHABLE; + } + case s_n_llhttp__internal__n_span_start_llhttp__on_protocol_1: + s_n_llhttp__internal__n_span_start_llhttp__on_protocol_1: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_protocol_1; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_protocol; + goto s_n_llhttp__internal__n_res_after_start; + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_llhttp__on_method_complete: + s_n_llhttp__internal__n_invoke_llhttp__on_method_complete: { + switch (llhttp__on_method_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_req_first_space_before_url; + case 21: + goto s_n_llhttp__internal__n_pause_26; + default: + goto s_n_llhttp__internal__n_error_1; + } + UNREACHABLE; } case s_n_llhttp__internal__n_req_or_res_method_2: s_n_llhttp__internal__n_req_or_res_method_2: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_req_or_res_method_2; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob60, 2); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob61, 2); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { @@ -12983,35 +6777,41 @@ static llparse_state_t llhttp__internal__run( return s_n_llhttp__internal__n_req_or_res_method_2; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_48; + goto s_n_llhttp__internal__n_error_110; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_update_type_1: + s_n_llhttp__internal__n_invoke_update_type_1: { + switch (llhttp__internal__c_update_type_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_version_1; + } + UNREACHABLE; } case s_n_llhttp__internal__n_req_or_res_method_3: s_n_llhttp__internal__n_req_or_res_method_3: { llparse_match_t match_seq; - + if (p == endp) { return s_n_llhttp__internal__n_req_or_res_method_3; } - match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob61, 3); + match_seq = llparse__match_sequence_id(state, p, endp, llparse_blob62, 3); p = match_seq.current; switch (match_seq.status) { case kMatchComplete: { p++; - goto s_n_llhttp__internal__n_invoke_update_type_1; + goto s_n_llhttp__internal__n_span_end_llhttp__on_method_1; } case kMatchPause: { return s_n_llhttp__internal__n_req_or_res_method_3; } case kMatchMismatch: { - goto s_n_llhttp__internal__n_error_48; + goto s_n_llhttp__internal__n_error_110; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_req_or_res_method_1: s_n_llhttp__internal__n_req_or_res_method_1: { @@ -13028,11 +6828,10 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_req_or_res_method_3; } default: { - goto s_n_llhttp__internal__n_error_48; + goto s_n_llhttp__internal__n_error_110; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_req_or_res_method: s_n_llhttp__internal__n_req_or_res_method: { @@ -13045,11 +6844,20 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_req_or_res_method_1; } default: { - goto s_n_llhttp__internal__n_error_48; + goto s_n_llhttp__internal__n_error_110; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_span_start_llhttp__on_method: + s_n_llhttp__internal__n_span_start_llhttp__on_method: { + if (p == endp) { + return s_n_llhttp__internal__n_span_start_llhttp__on_method; + } + state->_span_pos0 = (void*) p; + state->_span_cb0 = llhttp__on_method; + goto s_n_llhttp__internal__n_req_or_res_method; + UNREACHABLE; } case s_n_llhttp__internal__n_start_req_or_res: s_n_llhttp__internal__n_start_req_or_res: { @@ -13058,27 +6866,33 @@ static llparse_state_t llhttp__internal__run( } switch (*p) { case 'H': { - goto s_n_llhttp__internal__n_req_or_res_method; + goto s_n_llhttp__internal__n_span_start_llhttp__on_method; } default: { goto s_n_llhttp__internal__n_invoke_update_type_2; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } case s_n_llhttp__internal__n_invoke_load_type: s_n_llhttp__internal__n_invoke_load_type: { switch (llhttp__internal__c_load_type(state, p, endp)) { case 1: - goto s_n_llhttp__internal__n_start_req; + goto s_n_llhttp__internal__n_span_start_llhttp__on_method_1; case 2: - goto s_n_llhttp__internal__n_start_res; + goto s_n_llhttp__internal__n_span_start_llhttp__on_protocol_1; default: goto s_n_llhttp__internal__n_start_req_or_res; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + case s_n_llhttp__internal__n_invoke_update_finish: + s_n_llhttp__internal__n_invoke_update_finish: { + switch (llhttp__internal__c_update_finish(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_begin; + } + UNREACHABLE; } case s_n_llhttp__internal__n_start: s_n_llhttp__internal__n_start: { @@ -13095,107 +6909,118 @@ static llparse_state_t llhttp__internal__run( goto s_n_llhttp__internal__n_start; } default: { - goto s_n_llhttp__internal__n_invoke_update_finish; + goto s_n_llhttp__internal__n_invoke_load_initial_message_completed; } } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } default: - /* UNREACHABLE */ - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_37: { + s_n_llhttp__internal__n_error_2: { state->error = 0x7; state->reason = "Invalid characters in url"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_update_finish_2: { switch (llhttp__internal__c_update_finish_1(state, p, endp)) { default: goto s_n_llhttp__internal__n_start; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_test_lenient_flags: { - switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { - case 1: + s_n_llhttp__internal__n_invoke_update_initial_message_completed: { + switch (llhttp__internal__c_update_initial_message_completed(state, p, endp)) { + default: goto s_n_llhttp__internal__n_invoke_update_finish_2; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_update_content_length: { + switch (llhttp__internal__c_update_content_length(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_initial_message_completed; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_8: { + state->error = 0x5; + state->reason = "Data after `Connection: close`"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_3: { + switch (llhttp__internal__c_test_lenient_flags_3(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_closed; + default: + goto s_n_llhttp__internal__n_error_8; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_2: { + switch (llhttp__internal__c_test_lenient_flags_2(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_update_initial_message_completed; default: goto s_n_llhttp__internal__n_closed; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_update_finish_1: { switch (llhttp__internal__c_update_finish_1(state, p, endp)) { default: - goto s_n_llhttp__internal__n_invoke_test_lenient_flags; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_2; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_pause_5: { + s_n_llhttp__internal__n_pause_13: { state->error = 0x15; state->reason = "on_message_complete pause"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_is_equal_upgrade; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_9: { + s_n_llhttp__internal__n_error_38: { state->error = 0x12; state->reason = "`on_message_complete` callback error"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_pause_7: { + s_n_llhttp__internal__n_pause_15: { state->error = 0x15; state->reason = "on_chunk_complete pause"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_12: { + s_n_llhttp__internal__n_error_40: { state->error = 0x14; state->reason = "`on_chunk_complete` callback error"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_1: { switch (llhttp__on_chunk_complete(state, p, endp)) { case 0: goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; case 21: - goto s_n_llhttp__internal__n_pause_7; + goto s_n_llhttp__internal__n_pause_15; default: - goto s_n_llhttp__internal__n_error_12; + goto s_n_llhttp__internal__n_error_40; } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_error_11: { - state->error = 0x4; - state->reason = "Content-Length can't be present with Transfer-Encoding"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_pause_2: { state->error = 0x15; @@ -13203,17 +7028,15 @@ static llparse_state_t llhttp__internal__run( state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_pause_1; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_3: { + s_n_llhttp__internal__n_error_9: { state->error = 0x12; state->reason = "`on_message_complete` callback error"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_1: { switch (llhttp__on_message_complete(state, p, endp)) { @@ -13222,54 +7045,100 @@ static llparse_state_t llhttp__internal__run( case 21: goto s_n_llhttp__internal__n_pause_2; default: - goto s_n_llhttp__internal__n_error_3; + goto s_n_llhttp__internal__n_error_9; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_7: { + s_n_llhttp__internal__n_error_36: { state->error = 0xc; state->reason = "Chunk size overflow"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_error_10: { + state->error = 0xc; + state->reason = "Invalid character in chunk size"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_4: { + switch (llhttp__internal__c_test_lenient_flags_4(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_chunk_size_otherwise; + default: + goto s_n_llhttp__internal__n_error_10; + } + UNREACHABLE; } s_n_llhttp__internal__n_pause_3: { state->error = 0x15; state->reason = "on_chunk_complete pause"; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_update_content_length; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_update_content_length_1; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_5: { + s_n_llhttp__internal__n_error_14: { state->error = 0x14; state->reason = "`on_chunk_complete` callback error"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete: { switch (llhttp__on_chunk_complete(state, p, endp)) { case 0: - goto s_n_llhttp__internal__n_invoke_update_content_length; + goto s_n_llhttp__internal__n_invoke_update_content_length_1; case 21: goto s_n_llhttp__internal__n_pause_3; default: - goto s_n_llhttp__internal__n_error_5; + goto s_n_llhttp__internal__n_error_14; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_error_13: { + state->error = 0x19; + state->reason = "Missing expected CR after chunk data"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_6: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete; + default: + goto s_n_llhttp__internal__n_error_13; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_15: { + state->error = 0x2; + state->reason = "Expected LF after chunk data"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_7: { + switch (llhttp__internal__c_test_lenient_flags_7(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete; + default: + goto s_n_llhttp__internal__n_error_15; + } + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_body: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_body(state, start, p); @@ -13280,16 +7149,14 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_chunk_data_almost_done; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_or_flags: { switch (llhttp__internal__c_or_flags(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_field_start; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_pause_4: { state->error = 0x15; @@ -13297,17 +7164,15 @@ static llparse_state_t llhttp__internal__run( state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_is_equal_content_length; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_4: { + s_n_llhttp__internal__n_error_12: { state->error = 0x13; state->reason = "`on_chunk_header` callback error"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_llhttp__on_chunk_header: { switch (llhttp__on_chunk_header(state, p, endp)) { @@ -13316,43 +7181,470 @@ static llparse_state_t llhttp__internal__run( case 21: goto s_n_llhttp__internal__n_pause_4; default: - goto s_n_llhttp__internal__n_error_4; + goto s_n_llhttp__internal__n_error_12; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_6: { + s_n_llhttp__internal__n_error_16: { + state->error = 0x2; + state->reason = "Expected LF after chunk size"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_8: { + switch (llhttp__internal__c_test_lenient_flags_8(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_header; + default: + goto s_n_llhttp__internal__n_error_16; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_11: { + state->error = 0x19; + state->reason = "Missing expected CR after chunk size"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_5: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_chunk_size_almost_done; + default: + goto s_n_llhttp__internal__n_error_11; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_17: { + state->error = 0x2; + state->reason = "Invalid character in chunk extensions"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_18: { + state->error = 0x2; + state->reason = "Invalid character in chunk extensions"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_20: { + state->error = 0x19; + state->reason = "Missing expected CR after chunk extension name"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_5: { + state->error = 0x15; + state->reason = "on_chunk_extension_name pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_test_lenient_flags_9; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_19: { + state->error = 0x22; + state->reason = "`on_chunk_extension_name` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_name(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete; + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_6: { + state->error = 0x15; + state->reason = "on_chunk_extension_name pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_size_almost_done; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_21: { + state->error = 0x22; + state->reason = "`on_chunk_extension_name` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_name(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_1; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_1; + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_7: { + state->error = 0x15; + state->reason = "on_chunk_extension_name pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_extensions; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_22: { + state->error = 0x22; + state->reason = "`on_chunk_extension_name` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_name(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_2; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_2; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_25: { + state->error = 0x19; + state->reason = "Missing expected CR after chunk extension value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_8: { + state->error = 0x15; + state->reason = "on_chunk_extension_value pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_test_lenient_flags_10; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_24: { + state->error = 0x23; + state->reason = "`on_chunk_extension_value` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete; + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_9: { + state->error = 0x15; + state->reason = "on_chunk_extension_value pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_size_almost_done; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_26: { + state->error = 0x23; + state->reason = "`on_chunk_extension_value` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_1; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_28: { + state->error = 0x19; + state->reason = "Missing expected CR after chunk extension value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_11: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_chunk_size_almost_done; + default: + goto s_n_llhttp__internal__n_error_28; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_29: { + state->error = 0x2; + state->reason = "Invalid character in chunk extensions quote value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_10: { + state->error = 0x15; + state->reason = "on_chunk_extension_value pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_extension_quoted_value_done; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_27: { + state->error = 0x23; + state->reason = "`on_chunk_extension_value` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_2; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_3: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_30; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_error_30; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_4: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_31; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_error_31; + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_11: { + state->error = 0x15; + state->reason = "on_chunk_extension_value pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_extensions; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_32: { + state->error = 0x23; + state->reason = "`on_chunk_extension_value` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_5: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_3; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_value_complete_3; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_value_6: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_33; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_error_33; + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_12: { + state->error = 0x15; + state->reason = "on_chunk_extension_name pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_chunk_extension_value; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_23: { + state->error = 0x22; + state->reason = "`on_chunk_extension_name` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_extension_name_complete_3: { + switch (llhttp__on_chunk_extension_name_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_chunk_extension_value; + case 21: + goto s_n_llhttp__internal__n_pause_12; + default: + goto s_n_llhttp__internal__n_error_23; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_3: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_name(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_value; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_span_start_llhttp__on_chunk_extension_value; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_chunk_extension_name_4: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_chunk_extension_name(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_34; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_error_34; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_35: { state->error = 0xc; state->reason = "Invalid character in chunk size"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_mul_add_content_length: { switch (llhttp__internal__c_mul_add_content_length(state, p, endp, match)) { case 1: - goto s_n_llhttp__internal__n_error_7; + goto s_n_llhttp__internal__n_error_36; default: goto s_n_llhttp__internal__n_chunk_size; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_8: { + s_n_llhttp__internal__n_error_37: { state->error = 0xc; state->reason = "Invalid character in chunk size"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_body_1: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_body(state, start, p); @@ -13363,25 +7655,22 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_update_finish_3: { switch (llhttp__internal__c_update_finish_3(state, p, endp)) { default: goto s_n_llhttp__internal__n_span_start_llhttp__on_body_2; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_10: { + s_n_llhttp__internal__n_error_39: { state->error = 0xf; state->reason = "Request has invalid `Transfer-Encoding`"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_pause: { state->error = 0x15; @@ -13389,17 +7678,15 @@ static llparse_state_t llhttp__internal__run( state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_message_complete; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_2: { + s_n_llhttp__internal__n_error_7: { state->error = 0x12; state->reason = "`on_message_complete` callback error"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_llhttp__on_message_complete: { switch (llhttp__on_message_complete(state, p, endp)) { @@ -13408,52 +7695,46 @@ static llparse_state_t llhttp__internal__run( case 21: goto s_n_llhttp__internal__n_pause; default: - goto s_n_llhttp__internal__n_error_2; + goto s_n_llhttp__internal__n_error_7; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_or_flags_1: { switch (llhttp__internal__c_or_flags_1(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_or_flags_2: { switch (llhttp__internal__c_or_flags_1(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_update_upgrade: { switch (llhttp__internal__c_update_upgrade(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_or_flags_2; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_pause_6: { + s_n_llhttp__internal__n_pause_14: { state->error = 0x15; state->reason = "Paused by on_headers_complete"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_1: { + s_n_llhttp__internal__n_error_6: { state->error = 0x11; state->reason = "User callback error"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete: { switch (llhttp__on_headers_complete(state, p, endp)) { @@ -13464,64 +7745,236 @@ static llparse_state_t llhttp__internal__run( case 2: goto s_n_llhttp__internal__n_invoke_update_upgrade; case 21: - goto s_n_llhttp__internal__n_pause_6; + goto s_n_llhttp__internal__n_pause_14; default: - goto s_n_llhttp__internal__n_error_1; + goto s_n_llhttp__internal__n_error_6; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete: { switch (llhttp__before_headers_complete(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete; } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_test_lenient_flags_1: { - switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_error_11; - default: - goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_test_flags_1: { - switch (llhttp__internal__c_test_flags_1(state, p, endp)) { - case 1: - goto s_n_llhttp__internal__n_invoke_test_lenient_flags_1; - default: - goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete; - } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_test_flags: { switch (llhttp__internal__c_test_flags(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_1; default: - goto s_n_llhttp__internal__n_invoke_test_flags_1; + goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_13: { + s_n_llhttp__internal__n_invoke_test_lenient_flags_1: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_test_flags; + default: + goto s_n_llhttp__internal__n_error_5; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_17: { + state->error = 0x15; + state->reason = "on_chunk_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_42: { + state->error = 0x14; + state->reason = "`on_chunk_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_2: { + switch (llhttp__on_chunk_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__on_message_complete_2; + case 21: + goto s_n_llhttp__internal__n_pause_17; + default: + goto s_n_llhttp__internal__n_error_42; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_or_flags_3: { + switch (llhttp__internal__c_or_flags_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_or_flags_4: { + switch (llhttp__internal__c_or_flags_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_update_upgrade_1: { + switch (llhttp__internal__c_update_upgrade(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_or_flags_4; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_16: { + state->error = 0x15; + state->reason = "Paused by on_headers_complete"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_41: { + state->error = 0x11; + state->reason = "User callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete_1: { + switch (llhttp__on_headers_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_llhttp__after_headers_complete; + case 1: + goto s_n_llhttp__internal__n_invoke_or_flags_3; + case 2: + goto s_n_llhttp__internal__n_invoke_update_upgrade_1; + case 21: + goto s_n_llhttp__internal__n_pause_16; + default: + goto s_n_llhttp__internal__n_error_41; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete_1: { + switch (llhttp__before_headers_complete(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_headers_complete_1; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_flags_1: { + switch (llhttp__internal__c_test_flags(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_chunk_complete_2; + default: + goto s_n_llhttp__internal__n_invoke_llhttp__before_headers_complete_1; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_43: { + state->error = 0x2; + state->reason = "Expected LF after headers"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_12: { + switch (llhttp__internal__c_test_lenient_flags_8(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_test_flags_1; + default: + goto s_n_llhttp__internal__n_error_43; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_44: { + state->error = 0xa; + state->reason = "Invalid header token"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_field: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_field(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_5; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_error_5; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_13: { + switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_header_field_colon_discard_ws; + default: + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_field; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_60: { + state->error = 0xb; + state->reason = "Content-Length can't be present with Transfer-Encoding"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_47: { + state->error = 0xa; + state->reason = "Invalid header value char"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_15: { + switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_header_value_discard_ws; + default: + goto s_n_llhttp__internal__n_error_47; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_49: { state->error = 0xb; state->reason = "Empty Content-Length"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_18: { + state->error = 0x15; + state->reason = "on_header_value_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_field_start; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_48: { + state->error = 0x1d; + state->reason = "`on_header_value_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_header_value: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_header_value(state, start, p); @@ -13532,144 +7985,237 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_update_header_state: { switch (llhttp__internal__c_update_header_state(state, p, endp)) { default: goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_3: { - switch (llhttp__internal__c_or_flags_3(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_4: { - switch (llhttp__internal__c_or_flags_4(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state; - } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_or_flags_5: { switch (llhttp__internal__c_or_flags_5(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_update_header_state; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_or_flags_6: { switch (llhttp__internal__c_or_flags_6(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_or_flags_7: { + switch (llhttp__internal__c_or_flags_7(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_or_flags_8: { + switch (llhttp__internal__c_or_flags_8(state, p, endp)) { default: goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_header_state_2: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 5: + goto s_n_llhttp__internal__n_invoke_or_flags_5; + case 6: + goto s_n_llhttp__internal__n_invoke_or_flags_6; + case 7: + goto s_n_llhttp__internal__n_invoke_or_flags_7; + case 8: + goto s_n_llhttp__internal__n_invoke_or_flags_8; + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; + } + UNREACHABLE; } s_n_llhttp__internal__n_invoke_load_header_state_1: { - switch (llhttp__internal__c_load_header_state(state, p, endp)) { - case 5: - goto s_n_llhttp__internal__n_invoke_or_flags_3; - case 6: - goto s_n_llhttp__internal__n_invoke_or_flags_4; - case 7: - goto s_n_llhttp__internal__n_invoke_or_flags_5; - case 8: - goto s_n_llhttp__internal__n_invoke_or_flags_6; - default: - goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_load_header_state: { switch (llhttp__internal__c_load_header_state(state, p, endp)) { case 2: - goto s_n_llhttp__internal__n_error_13; + goto s_n_llhttp__internal__n_error_49; default: - goto s_n_llhttp__internal__n_invoke_load_header_state_1; + goto s_n_llhttp__internal__n_invoke_load_header_state_2; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_error_46: { + state->error = 0xa; + state->reason = "Invalid header value char"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_14: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_header_value_discard_lws; + default: + goto s_n_llhttp__internal__n_error_46; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_50: { + state->error = 0x2; + state->reason = "Expected LF after CR"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_16: { + switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_header_value_discard_lws; + default: + goto s_n_llhttp__internal__n_error_50; + } + UNREACHABLE; } s_n_llhttp__internal__n_invoke_update_header_state_1: { + switch (llhttp__internal__c_update_header_state_1(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_header_state_4: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 8: + goto s_n_llhttp__internal__n_invoke_update_header_state_1; + default: + goto s_n_llhttp__internal__n_span_start_llhttp__on_header_value_1; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_52: { + state->error = 0xa; + state->reason = "Unexpected whitespace after header value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_18: { + switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_load_header_state_4; + default: + goto s_n_llhttp__internal__n_error_52; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_update_header_state_2: { switch (llhttp__internal__c_update_header_state(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_7: { - switch (llhttp__internal__c_or_flags_3(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state_1; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_8: { - switch (llhttp__internal__c_or_flags_4(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state_1; - } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_or_flags_9: { switch (llhttp__internal__c_or_flags_5(state, p, endp)) { default: - goto s_n_llhttp__internal__n_invoke_update_header_state_1; + goto s_n_llhttp__internal__n_invoke_update_header_state_2; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_or_flags_10: { switch (llhttp__internal__c_or_flags_6(state, p, endp)) { default: - goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; + goto s_n_llhttp__internal__n_invoke_update_header_state_2; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_load_header_state_3: { - switch (llhttp__internal__c_load_header_state(state, p, endp)) { - case 5: - goto s_n_llhttp__internal__n_invoke_or_flags_7; - case 6: - goto s_n_llhttp__internal__n_invoke_or_flags_8; - case 7: - goto s_n_llhttp__internal__n_invoke_or_flags_9; - case 8: - goto s_n_llhttp__internal__n_invoke_or_flags_10; + s_n_llhttp__internal__n_invoke_or_flags_11: { + switch (llhttp__internal__c_or_flags_7(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_2; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_or_flags_12: { + switch (llhttp__internal__c_or_flags_8(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_14: { + s_n_llhttp__internal__n_invoke_load_header_state_5: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 5: + goto s_n_llhttp__internal__n_invoke_or_flags_9; + case 6: + goto s_n_llhttp__internal__n_invoke_or_flags_10; + case 7: + goto s_n_llhttp__internal__n_invoke_or_flags_11; + case 8: + goto s_n_llhttp__internal__n_invoke_or_flags_12; + default: + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_value_complete; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_53: { state->error = 0x3; state->reason = "Missing expected LF after header value"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_error_51: { + state->error = 0x19; + state->reason = "Missing expected CR after header value"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_header_value_1: { const unsigned char* start; int err; - + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_test_lenient_flags_17; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_17; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_almost_done; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_header_value_almost_done; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4: { + const unsigned char* start; + int err; + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_header_value(state, start, p); @@ -13680,13 +8226,12 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_header_value_almost_done; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_2: { + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_header_value(state, start, p); @@ -13698,283 +8243,376 @@ static llparse_state_t llhttp__internal__run( } p++; goto s_n_llhttp__internal__n_header_value_almost_done; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_header_value(state, start, p); if (err != 0) { state->error = err; - state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_header_value_almost_done; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_54; return s_error; } - p++; - goto s_n_llhttp__internal__n_header_value_almost_done; - /* UNREACHABLE */; - abort(); + goto s_n_llhttp__internal__n_error_54; + UNREACHABLE; } - s_n_llhttp__internal__n_error_15: { - state->error = 0xa; - state->reason = "Invalid header value char"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_test_lenient_flags_2: { - switch (llhttp__internal__c_test_lenient_flags_2(state, p, endp)) { + s_n_llhttp__internal__n_invoke_test_lenient_flags_19: { + switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_header_value_lenient; default: - goto s_n_llhttp__internal__n_error_15; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_3; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_update_header_state_3: { + s_n_llhttp__internal__n_invoke_update_header_state_4: { switch (llhttp__internal__c_update_header_state(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_value_connection; } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_11: { - switch (llhttp__internal__c_or_flags_3(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state_3; - } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_or_flags_12: { - switch (llhttp__internal__c_or_flags_4(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_invoke_update_header_state_3; - } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_or_flags_13: { switch (llhttp__internal__c_or_flags_5(state, p, endp)) { default: - goto s_n_llhttp__internal__n_invoke_update_header_state_3; + goto s_n_llhttp__internal__n_invoke_update_header_state_4; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_or_flags_14: { switch (llhttp__internal__c_or_flags_6(state, p, endp)) { default: - goto s_n_llhttp__internal__n_header_value_connection; + goto s_n_llhttp__internal__n_invoke_update_header_state_4; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_load_header_state_4: { - switch (llhttp__internal__c_load_header_state(state, p, endp)) { - case 5: - goto s_n_llhttp__internal__n_invoke_or_flags_11; - case 6: - goto s_n_llhttp__internal__n_invoke_or_flags_12; - case 7: - goto s_n_llhttp__internal__n_invoke_or_flags_13; - case 8: - goto s_n_llhttp__internal__n_invoke_or_flags_14; + s_n_llhttp__internal__n_invoke_or_flags_15: { + switch (llhttp__internal__c_or_flags_7(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_4; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_or_flags_16: { + switch (llhttp__internal__c_or_flags_8(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_value_connection; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_update_header_state_4: { - switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { + s_n_llhttp__internal__n_invoke_load_header_state_6: { + switch (llhttp__internal__c_load_header_state(state, p, endp)) { + case 5: + goto s_n_llhttp__internal__n_invoke_or_flags_13; + case 6: + goto s_n_llhttp__internal__n_invoke_or_flags_14; + case 7: + goto s_n_llhttp__internal__n_invoke_or_flags_15; + case 8: + goto s_n_llhttp__internal__n_invoke_or_flags_16; + default: + goto s_n_llhttp__internal__n_header_value_connection; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_update_header_state_5: { + switch (llhttp__internal__c_update_header_state_1(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_value_connection_token; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_update_header_state_2: { - switch (llhttp__internal__c_update_header_state_2(state, p, endp)) { + s_n_llhttp__internal__n_invoke_update_header_state_3: { + switch (llhttp__internal__c_update_header_state_3(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_value_connection_ws; } - /* UNREACHABLE */; - abort(); - } - s_n_llhttp__internal__n_invoke_update_header_state_5: { - switch (llhttp__internal__c_update_header_state_5(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_header_value_connection_ws; - } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_update_header_state_6: { switch (llhttp__internal__c_update_header_state_6(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_value_connection_ws; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4: { + s_n_llhttp__internal__n_invoke_update_header_state_7: { + switch (llhttp__internal__c_update_header_state_7(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_header_value_connection_ws; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_6: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_header_value(state, start, p); if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_17; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_56; return s_error; } - goto s_n_llhttp__internal__n_error_17; - /* UNREACHABLE */; - abort(); + goto s_n_llhttp__internal__n_error_56; + UNREACHABLE; } s_n_llhttp__internal__n_invoke_mul_add_content_length_1: { switch (llhttp__internal__c_mul_add_content_length_1(state, p, endp, match)) { case 1: - goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_4; + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_6; default: goto s_n_llhttp__internal__n_header_value_content_length; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_or_flags_15: { - switch (llhttp__internal__c_or_flags_15(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_17: { + switch (llhttp__internal__c_or_flags_17(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_value_otherwise; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_span_end_llhttp__on_header_value_5: { + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_7: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_header_value(state, start, p); if (err != 0) { state->error = err; state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_18; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_57; return s_error; } - goto s_n_llhttp__internal__n_error_18; - /* UNREACHABLE */; - abort(); + goto s_n_llhttp__internal__n_error_57; + UNREACHABLE; } - s_n_llhttp__internal__n_error_16: { + s_n_llhttp__internal__n_error_55: { state->error = 0x4; state->reason = "Duplicate Content-Length"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_test_flags_2: { switch (llhttp__internal__c_test_flags_2(state, p, endp)) { case 0: goto s_n_llhttp__internal__n_header_value_content_length; default: - goto s_n_llhttp__internal__n_error_16; + goto s_n_llhttp__internal__n_error_55; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_update_header_state_7: { - switch (llhttp__internal__c_update_header_state_7(state, p, endp)) { + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_9: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_59; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_error_59; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_update_header_state_8: { + switch (llhttp__internal__c_update_header_state_8(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_value_otherwise; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_update_header_state_8: { - switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { + s_n_llhttp__internal__n_span_end_llhttp__on_header_value_8: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_value(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_58; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_error_58; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_20: { + switch (llhttp__internal__c_test_lenient_flags_20(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_8; + default: + goto s_n_llhttp__internal__n_header_value_te_chunked; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_type_1: { + switch (llhttp__internal__c_load_type(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_20; + default: + goto s_n_llhttp__internal__n_header_value_te_chunked; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_update_header_state_9: { + switch (llhttp__internal__c_update_header_state_1(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_value; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_and_flags: { switch (llhttp__internal__c_and_flags(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_value_te_chunked; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_or_flags_16: { - switch (llhttp__internal__c_or_flags_16(state, p, endp)) { + s_n_llhttp__internal__n_invoke_or_flags_19: { + switch (llhttp__internal__c_or_flags_18(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_and_flags; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_or_flags_17: { - switch (llhttp__internal__c_or_flags_17(state, p, endp)) { + s_n_llhttp__internal__n_invoke_test_lenient_flags_21: { + switch (llhttp__internal__c_test_lenient_flags_20(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_span_end_llhttp__on_header_value_9; default: - goto s_n_llhttp__internal__n_invoke_update_header_state_8; + goto s_n_llhttp__internal__n_invoke_or_flags_19; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_load_header_state_2: { + s_n_llhttp__internal__n_invoke_load_type_2: { + switch (llhttp__internal__c_load_type(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_21; + default: + goto s_n_llhttp__internal__n_invoke_or_flags_19; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_or_flags_18: { + switch (llhttp__internal__c_or_flags_18(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_and_flags; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_flags_3: { + switch (llhttp__internal__c_test_flags_3(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_load_type_2; + default: + goto s_n_llhttp__internal__n_invoke_or_flags_18; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_or_flags_20: { + switch (llhttp__internal__c_or_flags_20(state, p, endp)) { + default: + goto s_n_llhttp__internal__n_invoke_update_header_state_9; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_header_state_3: { switch (llhttp__internal__c_load_header_state(state, p, endp)) { case 1: goto s_n_llhttp__internal__n_header_value_connection; case 2: goto s_n_llhttp__internal__n_invoke_test_flags_2; case 3: - goto s_n_llhttp__internal__n_invoke_or_flags_16; + goto s_n_llhttp__internal__n_invoke_test_flags_3; case 4: - goto s_n_llhttp__internal__n_invoke_or_flags_17; + goto s_n_llhttp__internal__n_invoke_or_flags_20; default: goto s_n_llhttp__internal__n_header_value; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_span_end_llhttp__on_header_field: { - const unsigned char* start; - int err; - - start = state->_span_pos0; - state->_span_pos0 = NULL; - err = llhttp__on_header_field(state, start, p); - if (err != 0) { - state->error = err; - state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; - return s_error; + s_n_llhttp__internal__n_invoke_test_lenient_flags_22: { + switch (llhttp__internal__c_test_lenient_flags_22(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_error_60; + default: + goto s_n_llhttp__internal__n_header_value_discard_ws; } - p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_flags_4: { + switch (llhttp__internal__c_test_flags_4(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_22; + default: + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_61: { + state->error = 0xf; + state->reason = "Transfer-Encoding can't be present with Content-Length"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_23: { + switch (llhttp__internal__c_test_lenient_flags_22(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_error_61; + default: + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_flags_5: { + switch (llhttp__internal__c_test_flags_2(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_23; + default: + goto s_n_llhttp__internal__n_header_value_discard_ws; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_19: { + state->error = 0x15; + state->reason = "on_header_field_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_load_header_state; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_45: { + state->error = 0x1c; + state->reason = "`on_header_field_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_header_field_1: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_header_field(state, start, p); @@ -13986,70 +8624,116 @@ static llparse_state_t llhttp__internal__run( } p++; goto s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_19: { + s_n_llhttp__internal__n_span_end_llhttp__on_header_field_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_header_field(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) (p + 1); + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; + return s_error; + } + p++; + goto s_n_llhttp__internal__n_invoke_llhttp__on_header_field_complete; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_62: { state->error = 0xa; state->reason = "Invalid header token"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_update_header_state_9: { - switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { + s_n_llhttp__internal__n_invoke_update_header_state_10: { + switch (llhttp__internal__c_update_header_state_1(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_field_general; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_store_header_state: { switch (llhttp__internal__c_store_header_state(state, p, endp, match)) { default: goto s_n_llhttp__internal__n_header_field_colon; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_update_header_state_10: { - switch (llhttp__internal__c_update_header_state_4(state, p, endp)) { + s_n_llhttp__internal__n_invoke_update_header_state_11: { + switch (llhttp__internal__c_update_header_state_1(state, p, endp)) { default: goto s_n_llhttp__internal__n_header_field_general; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_error_4: { + state->error = 0x1e; + state->reason = "Unexpected space after start line"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags: { + switch (llhttp__internal__c_test_lenient_flags(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_header_field_start; + default: + goto s_n_llhttp__internal__n_error_4; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_20: { + state->error = 0x15; + state->reason = "on_url_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_headers_start; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_3: { + state->error = 0x1a; + state->reason = "`on_url_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } s_n_llhttp__internal__n_invoke_llhttp__on_url_complete: { switch (llhttp__on_url_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_headers_start; + case 21: + goto s_n_llhttp__internal__n_pause_20; default: - goto s_n_llhttp__internal__n_header_field_start; + goto s_n_llhttp__internal__n_error_3; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_update_http_minor: { switch (llhttp__internal__c_update_http_minor(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_llhttp__on_url_complete; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_update_http_major: { switch (llhttp__internal__c_update_http_major(state, p, endp)) { default: goto s_n_llhttp__internal__n_invoke_update_http_minor; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_3: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14060,22 +8744,20 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_to_http09; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_20: { + s_n_llhttp__internal__n_error_63: { state->error = 0x7; state->reason = "Expected CRLF"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_4: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14086,261 +8768,402 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_lf_to_http09; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_23: { + s_n_llhttp__internal__n_error_72: { state->error = 0x17; state->reason = "Pause on PRI/Upgrade"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_24: { + s_n_llhttp__internal__n_error_73: { state->error = 0x9; state->reason = "Expected HTTP/2 Connection Preface"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_22: { + s_n_llhttp__internal__n_error_70: { + state->error = 0x2; + state->reason = "Expected CRLF after version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_26: { + switch (llhttp__internal__c_test_lenient_flags_8(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_headers_start; + default: + goto s_n_llhttp__internal__n_error_70; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_69: { state->error = 0x9; state->reason = "Expected CRLF after version"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_load_method_1: { - switch (llhttp__internal__c_load_method(state, p, endp)) { - case 34: - goto s_n_llhttp__internal__n_req_pri_upgrade; + s_n_llhttp__internal__n_invoke_test_lenient_flags_25: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_req_http_complete_crlf; default: - goto s_n_llhttp__internal__n_req_http_complete; + goto s_n_llhttp__internal__n_error_69; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_error_71: { + state->error = 0x9; + state->reason = "Expected CRLF after version"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_21: { + state->error = 0x15; + state->reason = "on_version_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_load_method_1; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_68: { + state->error = 0x21; + state->reason = "`on_version_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_version_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_version(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_version_complete; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_version_complete; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_version: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_version(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_67; + return s_error; + } + goto s_n_llhttp__internal__n_error_67; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_http_minor: { + switch (llhttp__internal__c_load_http_minor(state, p, endp)) { + case 9: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_1; + default: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_http_minor_1: { + switch (llhttp__internal__c_load_http_minor(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_1; + case 1: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_1; + default: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_http_minor_2: { + switch (llhttp__internal__c_load_http_minor(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_1; + default: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_http_major: { + switch (llhttp__internal__c_load_http_major(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_load_http_minor; + case 1: + goto s_n_llhttp__internal__n_invoke_load_http_minor_1; + case 2: + goto s_n_llhttp__internal__n_invoke_load_http_minor_2; + default: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_24: { + switch (llhttp__internal__c_test_lenient_flags_24(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_1; + default: + goto s_n_llhttp__internal__n_invoke_load_http_major; + } + UNREACHABLE; } s_n_llhttp__internal__n_invoke_store_http_minor: { switch (llhttp__internal__c_store_http_minor(state, p, endp, match)) { default: - goto s_n_llhttp__internal__n_invoke_load_method_1; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_24; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_25: { - state->error = 0x9; - state->reason = "Invalid minor version"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); + s_n_llhttp__internal__n_span_end_llhttp__on_version_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_version(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_74; + return s_error; + } + goto s_n_llhttp__internal__n_error_74; + UNREACHABLE; } - s_n_llhttp__internal__n_error_26: { - state->error = 0x9; - state->reason = "Expected dot"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); + s_n_llhttp__internal__n_span_end_llhttp__on_version_3: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_version(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_75; + return s_error; + } + goto s_n_llhttp__internal__n_error_75; + UNREACHABLE; } s_n_llhttp__internal__n_invoke_store_http_major: { switch (llhttp__internal__c_store_http_major(state, p, endp, match)) { default: goto s_n_llhttp__internal__n_req_http_dot; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_27: { - state->error = 0x9; - state->reason = "Invalid major version"; + s_n_llhttp__internal__n_span_end_llhttp__on_version_4: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_version(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_76; + return s_error; + } + goto s_n_llhttp__internal__n_error_76; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_77: { + state->error = 0x8; + state->reason = "Expected HTTP/, RTSP/ or ICE/"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_21: { + s_n_llhttp__internal__n_error_66: { state->error = 0x8; state->reason = "Invalid method for HTTP/x.x request"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_load_method: { - switch (llhttp__internal__c_load_method(state, p, endp)) { - case 0: - goto s_n_llhttp__internal__n_req_http_major; - case 1: - goto s_n_llhttp__internal__n_req_http_major; - case 2: - goto s_n_llhttp__internal__n_req_http_major; - case 3: - goto s_n_llhttp__internal__n_req_http_major; - case 4: - goto s_n_llhttp__internal__n_req_http_major; - case 5: - goto s_n_llhttp__internal__n_req_http_major; - case 6: - goto s_n_llhttp__internal__n_req_http_major; - case 7: - goto s_n_llhttp__internal__n_req_http_major; - case 8: - goto s_n_llhttp__internal__n_req_http_major; - case 9: - goto s_n_llhttp__internal__n_req_http_major; - case 10: - goto s_n_llhttp__internal__n_req_http_major; - case 11: - goto s_n_llhttp__internal__n_req_http_major; - case 12: - goto s_n_llhttp__internal__n_req_http_major; - case 13: - goto s_n_llhttp__internal__n_req_http_major; - case 14: - goto s_n_llhttp__internal__n_req_http_major; - case 15: - goto s_n_llhttp__internal__n_req_http_major; - case 16: - goto s_n_llhttp__internal__n_req_http_major; - case 17: - goto s_n_llhttp__internal__n_req_http_major; - case 18: - goto s_n_llhttp__internal__n_req_http_major; - case 19: - goto s_n_llhttp__internal__n_req_http_major; - case 20: - goto s_n_llhttp__internal__n_req_http_major; - case 21: - goto s_n_llhttp__internal__n_req_http_major; - case 22: - goto s_n_llhttp__internal__n_req_http_major; - case 23: - goto s_n_llhttp__internal__n_req_http_major; - case 24: - goto s_n_llhttp__internal__n_req_http_major; - case 25: - goto s_n_llhttp__internal__n_req_http_major; - case 26: - goto s_n_llhttp__internal__n_req_http_major; - case 27: - goto s_n_llhttp__internal__n_req_http_major; - case 28: - goto s_n_llhttp__internal__n_req_http_major; - case 29: - goto s_n_llhttp__internal__n_req_http_major; - case 30: - goto s_n_llhttp__internal__n_req_http_major; - case 31: - goto s_n_llhttp__internal__n_req_http_major; - case 32: - goto s_n_llhttp__internal__n_req_http_major; - case 33: - goto s_n_llhttp__internal__n_req_http_major; - case 34: - goto s_n_llhttp__internal__n_req_http_major; - default: - goto s_n_llhttp__internal__n_error_21; - } - /* UNREACHABLE */; - abort(); + s_n_llhttp__internal__n_pause_22: { + state->error = 0x15; + state->reason = "on_protocol_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_load_method; + return s_error; + UNREACHABLE; } - s_n_llhttp__internal__n_error_30: { - state->error = 0x8; - state->reason = "Expected HTTP/"; + s_n_llhttp__internal__n_error_65: { + state->error = 0x26; + state->reason = "`on_protocol_complete` callback error"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_28: { + s_n_llhttp__internal__n_span_end_llhttp__on_protocol: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_protocol(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_protocol_3: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_protocol(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_82; + return s_error; + } + goto s_n_llhttp__internal__n_error_82; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_79: { state->error = 0x8; state->reason = "Expected SOURCE method for ICE/x.x request"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_load_method_2: { - switch (llhttp__internal__c_load_method(state, p, endp)) { - case 33: - goto s_n_llhttp__internal__n_req_http_major; - default: - goto s_n_llhttp__internal__n_error_28; + s_n_llhttp__internal__n_pause_23: { + state->error = 0x15; + state->reason = "on_protocol_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_load_method_2; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_78: { + state->error = 0x26; + state->reason = "`on_protocol_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_protocol_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_protocol(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_1; + return s_error; } - /* UNREACHABLE */; - abort(); + goto s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_1; + UNREACHABLE; } - s_n_llhttp__internal__n_error_29: { + s_n_llhttp__internal__n_error_81: { state->error = 0x8; state->reason = "Invalid method for RTSP/x.x request"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_load_method_3: { - switch (llhttp__internal__c_load_method(state, p, endp)) { - case 1: - goto s_n_llhttp__internal__n_req_http_major; - case 3: - goto s_n_llhttp__internal__n_req_http_major; - case 6: - goto s_n_llhttp__internal__n_req_http_major; - case 35: - goto s_n_llhttp__internal__n_req_http_major; - case 36: - goto s_n_llhttp__internal__n_req_http_major; - case 37: - goto s_n_llhttp__internal__n_req_http_major; - case 38: - goto s_n_llhttp__internal__n_req_http_major; - case 39: - goto s_n_llhttp__internal__n_req_http_major; - case 40: - goto s_n_llhttp__internal__n_req_http_major; - case 41: - goto s_n_llhttp__internal__n_req_http_major; - case 42: - goto s_n_llhttp__internal__n_req_http_major; - case 43: - goto s_n_llhttp__internal__n_req_http_major; - case 44: - goto s_n_llhttp__internal__n_req_http_major; - case 45: - goto s_n_llhttp__internal__n_req_http_major; - default: - goto s_n_llhttp__internal__n_error_29; + s_n_llhttp__internal__n_pause_24: { + state->error = 0x15; + state->reason = "on_protocol_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_load_method_3; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_80: { + state->error = 0x26; + state->reason = "`on_protocol_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_protocol_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_protocol(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_2; + return s_error; } - /* UNREACHABLE */; - abort(); + goto s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_2; + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_25: { + state->error = 0x15; + state->reason = "on_url_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_req_http_start; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_64: { + state->error = 0x1a; + state->reason = "`on_url_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } s_n_llhttp__internal__n_invoke_llhttp__on_url_complete_1: { switch (llhttp__on_url_complete(state, p, endp)) { - default: + case 0: goto s_n_llhttp__internal__n_req_http_start; + case 21: + goto s_n_llhttp__internal__n_pause_25; + default: + goto s_n_llhttp__internal__n_error_64; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_5: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14351,13 +9174,12 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_to_http; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_6: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14368,13 +9190,12 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_to_http09; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_7: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14385,13 +9206,12 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_lf_to_http09; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_8: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14402,22 +9222,20 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_to_http; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_31: { + s_n_llhttp__internal__n_error_83: { state->error = 0x7; state->reason = "Invalid char in url fragment start"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_9: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14428,13 +9246,12 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_to_http09; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_10: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14445,13 +9262,12 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_lf_to_http09; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_11: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14462,31 +9278,28 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_to_http; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_32: { + s_n_llhttp__internal__n_error_84: { state->error = 0x7; state->reason = "Invalid char in url query"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_33: { + s_n_llhttp__internal__n_error_85: { state->error = 0x7; state->reason = "Invalid char in url path"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14497,13 +9310,12 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_to_http09; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_1: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14514,13 +9326,12 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_lf_to_http09; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_2: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14531,13 +9342,12 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_to_http; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_12: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14548,13 +9358,12 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_to_http09; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_13: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14565,13 +9374,12 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_lf_to_http09; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_url_14: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_url(state, start, p); @@ -14582,140 +9390,234 @@ static llparse_state_t llhttp__internal__run( return s_error; } goto s_n_llhttp__internal__n_url_skip_to_http; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_34: { + s_n_llhttp__internal__n_error_86: { state->error = 0x7; state->reason = "Double @ in url"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_35: { + s_n_llhttp__internal__n_error_87: { state->error = 0x7; state->reason = "Unexpected char in url server"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_36: { + s_n_llhttp__internal__n_error_88: { state->error = 0x7; state->reason = "Unexpected char in url server"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_38: { + s_n_llhttp__internal__n_error_89: { state->error = 0x7; state->reason = "Unexpected char in url schema"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_39: { + s_n_llhttp__internal__n_error_90: { state->error = 0x7; state->reason = "Unexpected char in url schema"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_40: { + s_n_llhttp__internal__n_error_91: { state->error = 0x7; state->reason = "Unexpected start char in url"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_is_equal_method: { switch (llhttp__internal__c_is_equal_method(state, p, endp)) { case 0: - goto s_n_llhttp__internal__n_span_start_llhttp__on_url_1; + goto s_n_llhttp__internal__n_url_entry_normal; default: - goto s_n_llhttp__internal__n_span_start_llhttp__on_url; + goto s_n_llhttp__internal__n_url_entry_connect; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_41: { + s_n_llhttp__internal__n_error_92: { state->error = 0x6; state->reason = "Expected space after method"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_29: { + state->error = 0x15; + state->reason = "on_method_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_req_first_space_before_url; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_111: { + state->error = 0x20; + state->reason = "`on_method_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_method_2: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_method(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_method_complete_1; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_method_complete_1; + UNREACHABLE; } s_n_llhttp__internal__n_invoke_store_method_1: { switch (llhttp__internal__c_store_method(state, p, endp, match)) { default: - goto s_n_llhttp__internal__n_req_first_space_before_url; + goto s_n_llhttp__internal__n_span_end_llhttp__on_method_2; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_49: { + s_n_llhttp__internal__n_error_112: { state->error = 0x6; state->reason = "Invalid method encountered"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_42: { + s_n_llhttp__internal__n_error_104: { state->error = 0xd; - state->reason = "Response overflow"; + state->reason = "Invalid status code"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_mul_add_status_code: { - switch (llhttp__internal__c_mul_add_status_code(state, p, endp, match)) { - case 1: - goto s_n_llhttp__internal__n_error_42; + s_n_llhttp__internal__n_error_102: { + state->error = 0xd; + state->reason = "Invalid status code"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_100: { + state->error = 0xd; + state->reason = "Invalid status code"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_27: { + state->error = 0x15; + state->reason = "on_status_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_headers_start; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_96: { + state->error = 0x1b; + state->reason = "`on_status_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_llhttp__on_status_complete: { + switch (llhttp__on_status_complete(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_headers_start; + case 21: + goto s_n_llhttp__internal__n_pause_27; default: - goto s_n_llhttp__internal__n_res_status_code; + goto s_n_llhttp__internal__n_error_96; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_error_95: { + state->error = 0xd; + state->reason = "Invalid response status"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_28: { + switch (llhttp__internal__c_test_lenient_flags_1(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + default: + goto s_n_llhttp__internal__n_error_95; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_97: { + state->error = 0x2; + state->reason = "Expected LF after CR"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_29: { + switch (llhttp__internal__c_test_lenient_flags_8(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + default: + goto s_n_llhttp__internal__n_error_97; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_98: { + state->error = 0x19; + state->reason = "Missing expected CR after response line"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_status: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_status(state, start, p); if (err != 0) { state->error = err; state->error_pos = (const char*) (p + 1); - state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_test_lenient_flags_30; return s_error; } p++; - goto s_n_llhttp__internal__n_invoke_llhttp__on_status_complete; - /* UNREACHABLE */; - abort(); + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_30; + UNREACHABLE; } s_n_llhttp__internal__n_span_end_llhttp__on_status_1: { const unsigned char* start; int err; - + start = state->_span_pos0; state->_span_pos0 = NULL; err = llhttp__on_status(state, start, p); @@ -14727,136 +9629,383 @@ static llparse_state_t llhttp__internal__run( } p++; goto s_n_llhttp__internal__n_res_line_almost_done; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_43: { + s_n_llhttp__internal__n_error_99: { state->error = 0xd; state->reason = "Invalid response status"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_mul_add_status_code_2: { + switch (llhttp__internal__c_mul_add_status_code(state, p, endp, match)) { + case 1: + goto s_n_llhttp__internal__n_error_100; + default: + goto s_n_llhttp__internal__n_res_status_code_otherwise; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_101: { + state->error = 0xd; + state->reason = "Invalid status code"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_mul_add_status_code_1: { + switch (llhttp__internal__c_mul_add_status_code(state, p, endp, match)) { + case 1: + goto s_n_llhttp__internal__n_error_102; + default: + goto s_n_llhttp__internal__n_res_status_code_digit_3; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_103: { + state->error = 0xd; + state->reason = "Invalid status code"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_mul_add_status_code: { + switch (llhttp__internal__c_mul_add_status_code(state, p, endp, match)) { + case 1: + goto s_n_llhttp__internal__n_error_104; + default: + goto s_n_llhttp__internal__n_res_status_code_digit_2; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_error_105: { + state->error = 0xd; + state->reason = "Invalid status code"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; } s_n_llhttp__internal__n_invoke_update_status_code: { switch (llhttp__internal__c_update_status_code(state, p, endp)) { default: - goto s_n_llhttp__internal__n_res_status_code; + goto s_n_llhttp__internal__n_res_status_code_digit_1; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_44: { + s_n_llhttp__internal__n_error_106: { state->error = 0x9; state->reason = "Expected space after version"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_28: { + state->error = 0x15; + state->reason = "on_version_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_res_after_version; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_94: { + state->error = 0x21; + state->reason = "`on_version_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_version_6: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_version(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_version_complete_1; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_version_complete_1; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_version_5: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_version(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_93; + return s_error; + } + goto s_n_llhttp__internal__n_error_93; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_http_minor_3: { + switch (llhttp__internal__c_load_http_minor(state, p, endp)) { + case 9: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_6; + default: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_5; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_http_minor_4: { + switch (llhttp__internal__c_load_http_minor(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_6; + case 1: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_6; + default: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_5; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_http_minor_5: { + switch (llhttp__internal__c_load_http_minor(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_6; + default: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_5; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_http_major_1: { + switch (llhttp__internal__c_load_http_major(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_load_http_minor_3; + case 1: + goto s_n_llhttp__internal__n_invoke_load_http_minor_4; + case 2: + goto s_n_llhttp__internal__n_invoke_load_http_minor_5; + default: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_5; + } + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_test_lenient_flags_27: { + switch (llhttp__internal__c_test_lenient_flags_24(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_span_end_llhttp__on_version_6; + default: + goto s_n_llhttp__internal__n_invoke_load_http_major_1; + } + UNREACHABLE; } s_n_llhttp__internal__n_invoke_store_http_minor_1: { switch (llhttp__internal__c_store_http_minor(state, p, endp, match)) { default: - goto s_n_llhttp__internal__n_res_http_end; + goto s_n_llhttp__internal__n_invoke_test_lenient_flags_27; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_45: { - state->error = 0x9; - state->reason = "Invalid minor version"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); + s_n_llhttp__internal__n_span_end_llhttp__on_version_7: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_version(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_107; + return s_error; + } + goto s_n_llhttp__internal__n_error_107; + UNREACHABLE; } - s_n_llhttp__internal__n_error_46: { - state->error = 0x9; - state->reason = "Expected dot"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); + s_n_llhttp__internal__n_span_end_llhttp__on_version_8: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_version(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_108; + return s_error; + } + goto s_n_llhttp__internal__n_error_108; + UNREACHABLE; } s_n_llhttp__internal__n_invoke_store_http_major_1: { switch (llhttp__internal__c_store_http_major(state, p, endp, match)) { default: goto s_n_llhttp__internal__n_res_http_dot; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_47: { - state->error = 0x9; - state->reason = "Invalid major version"; - state->error_pos = (const char*) p; - state->_current = (void*) (intptr_t) s_error; - return s_error; - /* UNREACHABLE */; - abort(); + s_n_llhttp__internal__n_span_end_llhttp__on_version_9: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_version(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_109; + return s_error; + } + goto s_n_llhttp__internal__n_error_109; + UNREACHABLE; } - s_n_llhttp__internal__n_error_50: { + s_n_llhttp__internal__n_error_114: { state->error = 0x8; - state->reason = "Expected HTTP/"; + state->reason = "Expected HTTP/, RTSP/ or ICE/"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_30: { + state->error = 0x15; + state->reason = "on_protocol_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_res_after_protocol; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_113: { + state->error = 0x26; + state->reason = "`on_protocol_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_protocol_4: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_protocol(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_3; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_protocol_complete_3; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_protocol_5: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_protocol(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_error_115; + return s_error; + } + goto s_n_llhttp__internal__n_error_115; + UNREACHABLE; + } + s_n_llhttp__internal__n_pause_26: { + state->error = 0x15; + state->reason = "on_method_complete pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_req_first_space_before_url; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_1: { + state->error = 0x20; + state->reason = "`on_method_complete` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_span_end_llhttp__on_method: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_method(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_llhttp__on_method_complete; + return s_error; + } + goto s_n_llhttp__internal__n_invoke_llhttp__on_method_complete; + UNREACHABLE; } s_n_llhttp__internal__n_invoke_update_type: { switch (llhttp__internal__c_update_type(state, p, endp)) { default: - goto s_n_llhttp__internal__n_req_first_space_before_url; + goto s_n_llhttp__internal__n_span_end_llhttp__on_method; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_store_method: { switch (llhttp__internal__c_store_method(state, p, endp, match)) { default: goto s_n_llhttp__internal__n_invoke_update_type; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_error_48: { + s_n_llhttp__internal__n_error_110: { state->error = 0x8; state->reason = "Invalid word encountered"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_update_type_1: { - switch (llhttp__internal__c_update_type_1(state, p, endp)) { - default: - goto s_n_llhttp__internal__n_res_http_major; + s_n_llhttp__internal__n_span_end_llhttp__on_method_1: { + const unsigned char* start; + int err; + + start = state->_span_pos0; + state->_span_pos0 = NULL; + err = llhttp__on_method(state, start, p); + if (err != 0) { + state->error = err; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_update_type_1; + return s_error; } - /* UNREACHABLE */; - abort(); + goto s_n_llhttp__internal__n_invoke_update_type_1; + UNREACHABLE; } s_n_llhttp__internal__n_invoke_update_type_2: { switch (llhttp__internal__c_update_type(state, p, endp)) { default: - goto s_n_llhttp__internal__n_start_req; + goto s_n_llhttp__internal__n_span_start_llhttp__on_method_1; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_pause_8: { + s_n_llhttp__internal__n_pause_31: { state->error = 0x15; state->reason = "on_message_begin pause"; state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_load_type; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_error: { state->error = 0x10; @@ -14864,28 +10013,54 @@ static llparse_state_t llhttp__internal__run( state->error_pos = (const char*) p; state->_current = (void*) (intptr_t) s_error; return s_error; - /* UNREACHABLE */; - abort(); + UNREACHABLE; } s_n_llhttp__internal__n_invoke_llhttp__on_message_begin: { switch (llhttp__on_message_begin(state, p, endp)) { case 0: goto s_n_llhttp__internal__n_invoke_load_type; case 21: - goto s_n_llhttp__internal__n_pause_8; + goto s_n_llhttp__internal__n_pause_31; default: goto s_n_llhttp__internal__n_error; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; } - s_n_llhttp__internal__n_invoke_update_finish: { - switch (llhttp__internal__c_update_finish(state, p, endp)) { + s_n_llhttp__internal__n_pause_32: { + state->error = 0x15; + state->reason = "on_reset pause"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_n_llhttp__internal__n_invoke_update_finish; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_error_116: { + state->error = 0x1f; + state->reason = "`on_reset` callback error"; + state->error_pos = (const char*) p; + state->_current = (void*) (intptr_t) s_error; + return s_error; + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_llhttp__on_reset: { + switch (llhttp__on_reset(state, p, endp)) { + case 0: + goto s_n_llhttp__internal__n_invoke_update_finish; + case 21: + goto s_n_llhttp__internal__n_pause_32; default: - goto s_n_llhttp__internal__n_invoke_llhttp__on_message_begin; + goto s_n_llhttp__internal__n_error_116; } - /* UNREACHABLE */; - abort(); + UNREACHABLE; + } + s_n_llhttp__internal__n_invoke_load_initial_message_completed: { + switch (llhttp__internal__c_load_initial_message_completed(state, p, endp)) { + case 1: + goto s_n_llhttp__internal__n_invoke_llhttp__on_reset; + default: + goto s_n_llhttp__internal__n_invoke_update_finish; + } + UNREACHABLE; } } @@ -14901,7 +10076,7 @@ int llhttp__internal_execute(llhttp__internal_t* state, const char* p, const cha if (state->_span_pos0 != NULL) { state->_span_pos0 = (void*) p; } - + next = llhttp__internal__run(state, (const unsigned char*) p, (const unsigned char*) endp); if (next == s_error) { return state->error; @@ -14911,7 +10086,7 @@ int llhttp__internal_execute(llhttp__internal_t* state, const char* p, const cha /* execute spans */ if (state->_span_pos0 != NULL) { int error; - + error = ((llhttp__internal__span_cb) state->_span_cb0)(state, state->_span_pos0, (const char*) endp); if (error != 0) { state->error = error; @@ -14919,8 +10094,6 @@ int llhttp__internal_execute(llhttp__internal_t* state, const char* p, const cha return error; } } - + return 0; -} - -#endif /* LLHTTP_STRICT_MODE */ +} \ No newline at end of file diff --git a/src/3rdparty/llhttp/llhttp.h b/src/3rdparty/llhttp/llhttp.h index 8f9590d3..59707a5d 100644 --- a/src/3rdparty/llhttp/llhttp.h +++ b/src/3rdparty/llhttp/llhttp.h @@ -1,14 +1,11 @@ + #ifndef INCLUDE_LLHTTP_H_ #define INCLUDE_LLHTTP_H_ -#define LLHTTP_VERSION_MAJOR 5 -#define LLHTTP_VERSION_MINOR 1 +#define LLHTTP_VERSION_MAJOR 9 +#define LLHTTP_VERSION_MINOR 3 #define LLHTTP_VERSION_PATCH 0 -#ifndef LLHTTP_STRICT_MODE -# define LLHTTP_STRICT_MODE 0 -#endif - #ifndef INCLUDE_LLHTTP_ITSELF_H_ #define INCLUDE_LLHTTP_ITSELF_H_ #ifdef __cplusplus @@ -33,11 +30,12 @@ struct llhttp__internal_s { uint8_t http_major; uint8_t http_minor; uint8_t header_state; - uint8_t lenient_flags; + uint16_t lenient_flags; uint8_t upgrade; uint8_t finish; uint16_t flags; uint16_t status_code; + uint8_t initial_message_completed; void* settings; }; @@ -49,6 +47,7 @@ int llhttp__internal_execute(llhttp__internal_t* s, const char* p, const char* e #endif #endif /* INCLUDE_LLHTTP_ITSELF_H_ */ + #ifndef LLLLHTTP_C_HEADERS_ #define LLLLHTTP_C_HEADERS_ #ifdef __cplusplus @@ -59,8 +58,10 @@ enum llhttp_errno { HPE_OK = 0, HPE_INTERNAL = 1, HPE_STRICT = 2, + HPE_CR_EXPECTED = 25, HPE_LF_EXPECTED = 3, HPE_UNEXPECTED_CONTENT_LENGTH = 4, + HPE_UNEXPECTED_SPACE = 30, HPE_CLOSED_CONNECTION = 5, HPE_INVALID_METHOD = 6, HPE_INVALID_URL = 7, @@ -80,7 +81,17 @@ enum llhttp_errno { HPE_PAUSED = 21, HPE_PAUSED_UPGRADE = 22, HPE_PAUSED_H2_UPGRADE = 23, - HPE_USER = 24 + HPE_USER = 24, + HPE_CB_URL_COMPLETE = 26, + HPE_CB_STATUS_COMPLETE = 27, + HPE_CB_METHOD_COMPLETE = 32, + HPE_CB_VERSION_COMPLETE = 33, + HPE_CB_HEADER_FIELD_COMPLETE = 28, + HPE_CB_HEADER_VALUE_COMPLETE = 29, + HPE_CB_CHUNK_EXTENSION_NAME_COMPLETE = 34, + HPE_CB_CHUNK_EXTENSION_VALUE_COMPLETE = 35, + HPE_CB_RESET = 31, + HPE_CB_PROTOCOL_COMPLETE = 38 }; typedef enum llhttp_errno llhttp_errno_t; @@ -100,7 +111,14 @@ typedef enum llhttp_flags llhttp_flags_t; enum llhttp_lenient_flags { LENIENT_HEADERS = 0x1, LENIENT_CHUNKED_LENGTH = 0x2, - LENIENT_KEEP_ALIVE = 0x4 + LENIENT_KEEP_ALIVE = 0x4, + LENIENT_TRANSFER_ENCODING = 0x8, + LENIENT_VERSION = 0x10, + LENIENT_DATA_AFTER_CLOSE = 0x20, + LENIENT_OPTIONAL_LF_AFTER_CR = 0x40, + LENIENT_OPTIONAL_CRLF_AFTER_CHUNK = 0x80, + LENIENT_OPTIONAL_CR_BEFORE_LF = 0x100, + LENIENT_SPACES_AFTER_CHUNK_SIZE = 0x200 }; typedef enum llhttp_lenient_flags llhttp_lenient_flags_t; @@ -164,16 +182,122 @@ enum llhttp_method { HTTP_SET_PARAMETER = 42, HTTP_REDIRECT = 43, HTTP_RECORD = 44, - HTTP_FLUSH = 45 + HTTP_FLUSH = 45, + HTTP_QUERY = 46 }; typedef enum llhttp_method llhttp_method_t; +enum llhttp_status { + HTTP_STATUS_CONTINUE = 100, + HTTP_STATUS_SWITCHING_PROTOCOLS = 101, + HTTP_STATUS_PROCESSING = 102, + HTTP_STATUS_EARLY_HINTS = 103, + HTTP_STATUS_RESPONSE_IS_STALE = 110, + HTTP_STATUS_REVALIDATION_FAILED = 111, + HTTP_STATUS_DISCONNECTED_OPERATION = 112, + HTTP_STATUS_HEURISTIC_EXPIRATION = 113, + HTTP_STATUS_MISCELLANEOUS_WARNING = 199, + HTTP_STATUS_OK = 200, + HTTP_STATUS_CREATED = 201, + HTTP_STATUS_ACCEPTED = 202, + HTTP_STATUS_NON_AUTHORITATIVE_INFORMATION = 203, + HTTP_STATUS_NO_CONTENT = 204, + HTTP_STATUS_RESET_CONTENT = 205, + HTTP_STATUS_PARTIAL_CONTENT = 206, + HTTP_STATUS_MULTI_STATUS = 207, + HTTP_STATUS_ALREADY_REPORTED = 208, + HTTP_STATUS_TRANSFORMATION_APPLIED = 214, + HTTP_STATUS_IM_USED = 226, + HTTP_STATUS_MISCELLANEOUS_PERSISTENT_WARNING = 299, + HTTP_STATUS_MULTIPLE_CHOICES = 300, + HTTP_STATUS_MOVED_PERMANENTLY = 301, + HTTP_STATUS_FOUND = 302, + HTTP_STATUS_SEE_OTHER = 303, + HTTP_STATUS_NOT_MODIFIED = 304, + HTTP_STATUS_USE_PROXY = 305, + HTTP_STATUS_SWITCH_PROXY = 306, + HTTP_STATUS_TEMPORARY_REDIRECT = 307, + HTTP_STATUS_PERMANENT_REDIRECT = 308, + HTTP_STATUS_BAD_REQUEST = 400, + HTTP_STATUS_UNAUTHORIZED = 401, + HTTP_STATUS_PAYMENT_REQUIRED = 402, + HTTP_STATUS_FORBIDDEN = 403, + HTTP_STATUS_NOT_FOUND = 404, + HTTP_STATUS_METHOD_NOT_ALLOWED = 405, + HTTP_STATUS_NOT_ACCEPTABLE = 406, + HTTP_STATUS_PROXY_AUTHENTICATION_REQUIRED = 407, + HTTP_STATUS_REQUEST_TIMEOUT = 408, + HTTP_STATUS_CONFLICT = 409, + HTTP_STATUS_GONE = 410, + HTTP_STATUS_LENGTH_REQUIRED = 411, + HTTP_STATUS_PRECONDITION_FAILED = 412, + HTTP_STATUS_PAYLOAD_TOO_LARGE = 413, + HTTP_STATUS_URI_TOO_LONG = 414, + HTTP_STATUS_UNSUPPORTED_MEDIA_TYPE = 415, + HTTP_STATUS_RANGE_NOT_SATISFIABLE = 416, + HTTP_STATUS_EXPECTATION_FAILED = 417, + HTTP_STATUS_IM_A_TEAPOT = 418, + HTTP_STATUS_PAGE_EXPIRED = 419, + HTTP_STATUS_ENHANCE_YOUR_CALM = 420, + HTTP_STATUS_MISDIRECTED_REQUEST = 421, + HTTP_STATUS_UNPROCESSABLE_ENTITY = 422, + HTTP_STATUS_LOCKED = 423, + HTTP_STATUS_FAILED_DEPENDENCY = 424, + HTTP_STATUS_TOO_EARLY = 425, + HTTP_STATUS_UPGRADE_REQUIRED = 426, + HTTP_STATUS_PRECONDITION_REQUIRED = 428, + HTTP_STATUS_TOO_MANY_REQUESTS = 429, + HTTP_STATUS_REQUEST_HEADER_FIELDS_TOO_LARGE_UNOFFICIAL = 430, + HTTP_STATUS_REQUEST_HEADER_FIELDS_TOO_LARGE = 431, + HTTP_STATUS_LOGIN_TIMEOUT = 440, + HTTP_STATUS_NO_RESPONSE = 444, + HTTP_STATUS_RETRY_WITH = 449, + HTTP_STATUS_BLOCKED_BY_PARENTAL_CONTROL = 450, + HTTP_STATUS_UNAVAILABLE_FOR_LEGAL_REASONS = 451, + HTTP_STATUS_CLIENT_CLOSED_LOAD_BALANCED_REQUEST = 460, + HTTP_STATUS_INVALID_X_FORWARDED_FOR = 463, + HTTP_STATUS_REQUEST_HEADER_TOO_LARGE = 494, + HTTP_STATUS_SSL_CERTIFICATE_ERROR = 495, + HTTP_STATUS_SSL_CERTIFICATE_REQUIRED = 496, + HTTP_STATUS_HTTP_REQUEST_SENT_TO_HTTPS_PORT = 497, + HTTP_STATUS_INVALID_TOKEN = 498, + HTTP_STATUS_CLIENT_CLOSED_REQUEST = 499, + HTTP_STATUS_INTERNAL_SERVER_ERROR = 500, + HTTP_STATUS_NOT_IMPLEMENTED = 501, + HTTP_STATUS_BAD_GATEWAY = 502, + HTTP_STATUS_SERVICE_UNAVAILABLE = 503, + HTTP_STATUS_GATEWAY_TIMEOUT = 504, + HTTP_STATUS_HTTP_VERSION_NOT_SUPPORTED = 505, + HTTP_STATUS_VARIANT_ALSO_NEGOTIATES = 506, + HTTP_STATUS_INSUFFICIENT_STORAGE = 507, + HTTP_STATUS_LOOP_DETECTED = 508, + HTTP_STATUS_BANDWIDTH_LIMIT_EXCEEDED = 509, + HTTP_STATUS_NOT_EXTENDED = 510, + HTTP_STATUS_NETWORK_AUTHENTICATION_REQUIRED = 511, + HTTP_STATUS_WEB_SERVER_UNKNOWN_ERROR = 520, + HTTP_STATUS_WEB_SERVER_IS_DOWN = 521, + HTTP_STATUS_CONNECTION_TIMEOUT = 522, + HTTP_STATUS_ORIGIN_IS_UNREACHABLE = 523, + HTTP_STATUS_TIMEOUT_OCCURED = 524, + HTTP_STATUS_SSL_HANDSHAKE_FAILED = 525, + HTTP_STATUS_INVALID_SSL_CERTIFICATE = 526, + HTTP_STATUS_RAILGUN_ERROR = 527, + HTTP_STATUS_SITE_IS_OVERLOADED = 529, + HTTP_STATUS_SITE_IS_FROZEN = 530, + HTTP_STATUS_IDENTITY_PROVIDER_AUTHENTICATION_ERROR = 561, + HTTP_STATUS_NETWORK_READ_TIMEOUT = 598, + HTTP_STATUS_NETWORK_CONNECT_TIMEOUT = 599 +}; +typedef enum llhttp_status llhttp_status_t; + #define HTTP_ERRNO_MAP(XX) \ XX(0, OK, OK) \ XX(1, INTERNAL, INTERNAL) \ XX(2, STRICT, STRICT) \ + XX(25, CR_EXPECTED, CR_EXPECTED) \ XX(3, LF_EXPECTED, LF_EXPECTED) \ XX(4, UNEXPECTED_CONTENT_LENGTH, UNEXPECTED_CONTENT_LENGTH) \ + XX(30, UNEXPECTED_SPACE, UNEXPECTED_SPACE) \ XX(5, CLOSED_CONNECTION, CLOSED_CONNECTION) \ XX(6, INVALID_METHOD, INVALID_METHOD) \ XX(7, INVALID_URL, INVALID_URL) \ @@ -194,9 +318,74 @@ typedef enum llhttp_method llhttp_method_t; XX(22, PAUSED_UPGRADE, PAUSED_UPGRADE) \ XX(23, PAUSED_H2_UPGRADE, PAUSED_H2_UPGRADE) \ XX(24, USER, USER) \ + XX(26, CB_URL_COMPLETE, CB_URL_COMPLETE) \ + XX(27, CB_STATUS_COMPLETE, CB_STATUS_COMPLETE) \ + XX(32, CB_METHOD_COMPLETE, CB_METHOD_COMPLETE) \ + XX(33, CB_VERSION_COMPLETE, CB_VERSION_COMPLETE) \ + XX(28, CB_HEADER_FIELD_COMPLETE, CB_HEADER_FIELD_COMPLETE) \ + XX(29, CB_HEADER_VALUE_COMPLETE, CB_HEADER_VALUE_COMPLETE) \ + XX(34, CB_CHUNK_EXTENSION_NAME_COMPLETE, CB_CHUNK_EXTENSION_NAME_COMPLETE) \ + XX(35, CB_CHUNK_EXTENSION_VALUE_COMPLETE, CB_CHUNK_EXTENSION_VALUE_COMPLETE) \ + XX(31, CB_RESET, CB_RESET) \ + XX(38, CB_PROTOCOL_COMPLETE, CB_PROTOCOL_COMPLETE) \ #define HTTP_METHOD_MAP(XX) \ + XX(0, DELETE, DELETE) \ + XX(1, GET, GET) \ + XX(2, HEAD, HEAD) \ + XX(3, POST, POST) \ + XX(4, PUT, PUT) \ + XX(5, CONNECT, CONNECT) \ + XX(6, OPTIONS, OPTIONS) \ + XX(7, TRACE, TRACE) \ + XX(8, COPY, COPY) \ + XX(9, LOCK, LOCK) \ + XX(10, MKCOL, MKCOL) \ + XX(11, MOVE, MOVE) \ + XX(12, PROPFIND, PROPFIND) \ + XX(13, PROPPATCH, PROPPATCH) \ + XX(14, SEARCH, SEARCH) \ + XX(15, UNLOCK, UNLOCK) \ + XX(16, BIND, BIND) \ + XX(17, REBIND, REBIND) \ + XX(18, UNBIND, UNBIND) \ + XX(19, ACL, ACL) \ + XX(20, REPORT, REPORT) \ + XX(21, MKACTIVITY, MKACTIVITY) \ + XX(22, CHECKOUT, CHECKOUT) \ + XX(23, MERGE, MERGE) \ + XX(24, MSEARCH, M-SEARCH) \ + XX(25, NOTIFY, NOTIFY) \ + XX(26, SUBSCRIBE, SUBSCRIBE) \ + XX(27, UNSUBSCRIBE, UNSUBSCRIBE) \ + XX(28, PATCH, PATCH) \ + XX(29, PURGE, PURGE) \ + XX(30, MKCALENDAR, MKCALENDAR) \ + XX(31, LINK, LINK) \ + XX(32, UNLINK, UNLINK) \ + XX(33, SOURCE, SOURCE) \ + XX(46, QUERY, QUERY) \ + + +#define RTSP_METHOD_MAP(XX) \ + XX(1, GET, GET) \ + XX(3, POST, POST) \ + XX(6, OPTIONS, OPTIONS) \ + XX(35, DESCRIBE, DESCRIBE) \ + XX(36, ANNOUNCE, ANNOUNCE) \ + XX(37, SETUP, SETUP) \ + XX(38, PLAY, PLAY) \ + XX(39, PAUSE, PAUSE) \ + XX(40, TEARDOWN, TEARDOWN) \ + XX(41, GET_PARAMETER, GET_PARAMETER) \ + XX(42, SET_PARAMETER, SET_PARAMETER) \ + XX(43, REDIRECT, REDIRECT) \ + XX(44, RECORD, RECORD) \ + XX(45, FLUSH, FLUSH) \ + + +#define HTTP_ALL_METHOD_MAP(XX) \ XX(0, DELETE, DELETE) \ XX(1, GET, GET) \ XX(2, HEAD, HEAD) \ @@ -243,14 +432,117 @@ typedef enum llhttp_method llhttp_method_t; XX(43, REDIRECT, REDIRECT) \ XX(44, RECORD, RECORD) \ XX(45, FLUSH, FLUSH) \ + XX(46, QUERY, QUERY) \ +#define HTTP_STATUS_MAP(XX) \ + XX(100, CONTINUE, CONTINUE) \ + XX(101, SWITCHING_PROTOCOLS, SWITCHING_PROTOCOLS) \ + XX(102, PROCESSING, PROCESSING) \ + XX(103, EARLY_HINTS, EARLY_HINTS) \ + XX(110, RESPONSE_IS_STALE, RESPONSE_IS_STALE) \ + XX(111, REVALIDATION_FAILED, REVALIDATION_FAILED) \ + XX(112, DISCONNECTED_OPERATION, DISCONNECTED_OPERATION) \ + XX(113, HEURISTIC_EXPIRATION, HEURISTIC_EXPIRATION) \ + XX(199, MISCELLANEOUS_WARNING, MISCELLANEOUS_WARNING) \ + XX(200, OK, OK) \ + XX(201, CREATED, CREATED) \ + XX(202, ACCEPTED, ACCEPTED) \ + XX(203, NON_AUTHORITATIVE_INFORMATION, NON_AUTHORITATIVE_INFORMATION) \ + XX(204, NO_CONTENT, NO_CONTENT) \ + XX(205, RESET_CONTENT, RESET_CONTENT) \ + XX(206, PARTIAL_CONTENT, PARTIAL_CONTENT) \ + XX(207, MULTI_STATUS, MULTI_STATUS) \ + XX(208, ALREADY_REPORTED, ALREADY_REPORTED) \ + XX(214, TRANSFORMATION_APPLIED, TRANSFORMATION_APPLIED) \ + XX(226, IM_USED, IM_USED) \ + XX(299, MISCELLANEOUS_PERSISTENT_WARNING, MISCELLANEOUS_PERSISTENT_WARNING) \ + XX(300, MULTIPLE_CHOICES, MULTIPLE_CHOICES) \ + XX(301, MOVED_PERMANENTLY, MOVED_PERMANENTLY) \ + XX(302, FOUND, FOUND) \ + XX(303, SEE_OTHER, SEE_OTHER) \ + XX(304, NOT_MODIFIED, NOT_MODIFIED) \ + XX(305, USE_PROXY, USE_PROXY) \ + XX(306, SWITCH_PROXY, SWITCH_PROXY) \ + XX(307, TEMPORARY_REDIRECT, TEMPORARY_REDIRECT) \ + XX(308, PERMANENT_REDIRECT, PERMANENT_REDIRECT) \ + XX(400, BAD_REQUEST, BAD_REQUEST) \ + XX(401, UNAUTHORIZED, UNAUTHORIZED) \ + XX(402, PAYMENT_REQUIRED, PAYMENT_REQUIRED) \ + XX(403, FORBIDDEN, FORBIDDEN) \ + XX(404, NOT_FOUND, NOT_FOUND) \ + XX(405, METHOD_NOT_ALLOWED, METHOD_NOT_ALLOWED) \ + XX(406, NOT_ACCEPTABLE, NOT_ACCEPTABLE) \ + XX(407, PROXY_AUTHENTICATION_REQUIRED, PROXY_AUTHENTICATION_REQUIRED) \ + XX(408, REQUEST_TIMEOUT, REQUEST_TIMEOUT) \ + XX(409, CONFLICT, CONFLICT) \ + XX(410, GONE, GONE) \ + XX(411, LENGTH_REQUIRED, LENGTH_REQUIRED) \ + XX(412, PRECONDITION_FAILED, PRECONDITION_FAILED) \ + XX(413, PAYLOAD_TOO_LARGE, PAYLOAD_TOO_LARGE) \ + XX(414, URI_TOO_LONG, URI_TOO_LONG) \ + XX(415, UNSUPPORTED_MEDIA_TYPE, UNSUPPORTED_MEDIA_TYPE) \ + XX(416, RANGE_NOT_SATISFIABLE, RANGE_NOT_SATISFIABLE) \ + XX(417, EXPECTATION_FAILED, EXPECTATION_FAILED) \ + XX(418, IM_A_TEAPOT, IM_A_TEAPOT) \ + XX(419, PAGE_EXPIRED, PAGE_EXPIRED) \ + XX(420, ENHANCE_YOUR_CALM, ENHANCE_YOUR_CALM) \ + XX(421, MISDIRECTED_REQUEST, MISDIRECTED_REQUEST) \ + XX(422, UNPROCESSABLE_ENTITY, UNPROCESSABLE_ENTITY) \ + XX(423, LOCKED, LOCKED) \ + XX(424, FAILED_DEPENDENCY, FAILED_DEPENDENCY) \ + XX(425, TOO_EARLY, TOO_EARLY) \ + XX(426, UPGRADE_REQUIRED, UPGRADE_REQUIRED) \ + XX(428, PRECONDITION_REQUIRED, PRECONDITION_REQUIRED) \ + XX(429, TOO_MANY_REQUESTS, TOO_MANY_REQUESTS) \ + XX(430, REQUEST_HEADER_FIELDS_TOO_LARGE_UNOFFICIAL, REQUEST_HEADER_FIELDS_TOO_LARGE_UNOFFICIAL) \ + XX(431, REQUEST_HEADER_FIELDS_TOO_LARGE, REQUEST_HEADER_FIELDS_TOO_LARGE) \ + XX(440, LOGIN_TIMEOUT, LOGIN_TIMEOUT) \ + XX(444, NO_RESPONSE, NO_RESPONSE) \ + XX(449, RETRY_WITH, RETRY_WITH) \ + XX(450, BLOCKED_BY_PARENTAL_CONTROL, BLOCKED_BY_PARENTAL_CONTROL) \ + XX(451, UNAVAILABLE_FOR_LEGAL_REASONS, UNAVAILABLE_FOR_LEGAL_REASONS) \ + XX(460, CLIENT_CLOSED_LOAD_BALANCED_REQUEST, CLIENT_CLOSED_LOAD_BALANCED_REQUEST) \ + XX(463, INVALID_X_FORWARDED_FOR, INVALID_X_FORWARDED_FOR) \ + XX(494, REQUEST_HEADER_TOO_LARGE, REQUEST_HEADER_TOO_LARGE) \ + XX(495, SSL_CERTIFICATE_ERROR, SSL_CERTIFICATE_ERROR) \ + XX(496, SSL_CERTIFICATE_REQUIRED, SSL_CERTIFICATE_REQUIRED) \ + XX(497, HTTP_REQUEST_SENT_TO_HTTPS_PORT, HTTP_REQUEST_SENT_TO_HTTPS_PORT) \ + XX(498, INVALID_TOKEN, INVALID_TOKEN) \ + XX(499, CLIENT_CLOSED_REQUEST, CLIENT_CLOSED_REQUEST) \ + XX(500, INTERNAL_SERVER_ERROR, INTERNAL_SERVER_ERROR) \ + XX(501, NOT_IMPLEMENTED, NOT_IMPLEMENTED) \ + XX(502, BAD_GATEWAY, BAD_GATEWAY) \ + XX(503, SERVICE_UNAVAILABLE, SERVICE_UNAVAILABLE) \ + XX(504, GATEWAY_TIMEOUT, GATEWAY_TIMEOUT) \ + XX(505, HTTP_VERSION_NOT_SUPPORTED, HTTP_VERSION_NOT_SUPPORTED) \ + XX(506, VARIANT_ALSO_NEGOTIATES, VARIANT_ALSO_NEGOTIATES) \ + XX(507, INSUFFICIENT_STORAGE, INSUFFICIENT_STORAGE) \ + XX(508, LOOP_DETECTED, LOOP_DETECTED) \ + XX(509, BANDWIDTH_LIMIT_EXCEEDED, BANDWIDTH_LIMIT_EXCEEDED) \ + XX(510, NOT_EXTENDED, NOT_EXTENDED) \ + XX(511, NETWORK_AUTHENTICATION_REQUIRED, NETWORK_AUTHENTICATION_REQUIRED) \ + XX(520, WEB_SERVER_UNKNOWN_ERROR, WEB_SERVER_UNKNOWN_ERROR) \ + XX(521, WEB_SERVER_IS_DOWN, WEB_SERVER_IS_DOWN) \ + XX(522, CONNECTION_TIMEOUT, CONNECTION_TIMEOUT) \ + XX(523, ORIGIN_IS_UNREACHABLE, ORIGIN_IS_UNREACHABLE) \ + XX(524, TIMEOUT_OCCURED, TIMEOUT_OCCURED) \ + XX(525, SSL_HANDSHAKE_FAILED, SSL_HANDSHAKE_FAILED) \ + XX(526, INVALID_SSL_CERTIFICATE, INVALID_SSL_CERTIFICATE) \ + XX(527, RAILGUN_ERROR, RAILGUN_ERROR) \ + XX(529, SITE_IS_OVERLOADED, SITE_IS_OVERLOADED) \ + XX(530, SITE_IS_FROZEN, SITE_IS_FROZEN) \ + XX(561, IDENTITY_PROVIDER_AUTHENTICATION_ERROR, IDENTITY_PROVIDER_AUTHENTICATION_ERROR) \ + XX(598, NETWORK_READ_TIMEOUT, NETWORK_READ_TIMEOUT) \ + XX(599, NETWORK_CONNECT_TIMEOUT, NETWORK_CONNECT_TIMEOUT) \ + #ifdef __cplusplus } /* extern "C" */ #endif #endif /* LLLLHTTP_C_HEADERS_ */ + #ifndef INCLUDE_LLHTTP_API_H_ #define INCLUDE_LLHTTP_API_H_ #ifdef __cplusplus @@ -274,10 +566,16 @@ struct llhttp_settings_s { /* Possible return values 0, -1, `HPE_PAUSED` */ llhttp_cb on_message_begin; + /* Possible return values 0, -1, HPE_USER */ + llhttp_data_cb on_protocol; llhttp_data_cb on_url; llhttp_data_cb on_status; + llhttp_data_cb on_method; + llhttp_data_cb on_version; llhttp_data_cb on_header_field; llhttp_data_cb on_header_value; + llhttp_data_cb on_chunk_extension_name; + llhttp_data_cb on_chunk_extension_value; /* Possible return values: * 0 - Proceed normally @@ -290,10 +588,20 @@ struct llhttp_settings_s { */ llhttp_cb on_headers_complete; + /* Possible return values 0, -1, HPE_USER */ llhttp_data_cb on_body; /* Possible return values 0, -1, `HPE_PAUSED` */ llhttp_cb on_message_complete; + llhttp_cb on_protocol_complete; + llhttp_cb on_url_complete; + llhttp_cb on_status_complete; + llhttp_cb on_method_complete; + llhttp_cb on_version_complete; + llhttp_cb on_header_field_complete; + llhttp_cb on_header_value_complete; + llhttp_cb on_chunk_extension_name_complete; + llhttp_cb on_chunk_extension_value_complete; /* When on_chunk_header is called, the current chunk length is stored * in parser->content_length. @@ -301,11 +609,7 @@ struct llhttp_settings_s { */ llhttp_cb on_chunk_header; llhttp_cb on_chunk_complete; - - llhttp_cb on_url_complete; - llhttp_cb on_status_complete; - llhttp_cb on_header_field_complete; - llhttp_cb on_header_value_complete; + llhttp_cb on_reset; }; /* Initialize the parser with specific type and user settings. @@ -318,8 +622,6 @@ LLHTTP_EXPORT void llhttp_init(llhttp_t* parser, llhttp_type_t type, const llhttp_settings_t* settings); -#if defined(__wasm__) - LLHTTP_EXPORT llhttp_t* llhttp_alloc(llhttp_type_t type); @@ -344,8 +646,6 @@ int llhttp_get_status_code(llhttp_t* parser); LLHTTP_EXPORT uint8_t llhttp_get_upgrade(llhttp_t* parser); -#endif // defined(__wasm__) - /* Reset an already initialized parser back to the start state, preserving the * existing parser type, callback settings, user data, and lenient flags. */ @@ -459,6 +759,9 @@ const char* llhttp_errno_name(llhttp_errno_t err); LLHTTP_EXPORT const char* llhttp_method_name(llhttp_method_t method); +/* Returns textual name of HTTP status */ +LLHTTP_EXPORT +const char* llhttp_status_name(llhttp_status_t status); /* Enables/disables lenient header value parsing (disabled by default). * @@ -467,7 +770,8 @@ const char* llhttp_method_name(llhttp_method_t method); * `HPE_INVALID_HEADER_TOKEN` will be raised for incorrect header values when * lenient parsing is "on". * - * **(USE AT YOUR OWN RISK)** + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** */ LLHTTP_EXPORT void llhttp_set_lenient_headers(llhttp_t* parser, int enabled); @@ -481,7 +785,8 @@ void llhttp_set_lenient_headers(llhttp_t* parser, int enabled); * request smuggling, but may be less desirable for small number of cases * involving legacy servers. * - * **(USE AT YOUR OWN RISK)** + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** */ LLHTTP_EXPORT void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled); @@ -496,13 +801,105 @@ void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled); * but might interact badly with outdated and insecure clients. With this flag * the extra request/response will be parsed normally. * - * **(USE AT YOUR OWN RISK)** + * **Enabling this flag can pose a security issue since you will be exposed to + * poisoning attacks. USE WITH CAUTION!** */ +LLHTTP_EXPORT void llhttp_set_lenient_keep_alive(llhttp_t* parser, int enabled); +/* Enables/disables lenient handling of `Transfer-Encoding` header. + * + * Normally `llhttp` would error when a `Transfer-Encoding` has `chunked` value + * and another value after it (either in a single header or in multiple + * headers whose value are internally joined using `, `). + * This is mandated by the spec to reliably determine request body size and thus + * avoid request smuggling. + * With this flag the extra value will be parsed normally. + * + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_transfer_encoding(llhttp_t* parser, int enabled); + +/* Enables/disables lenient handling of HTTP version. + * + * Normally `llhttp` would error when the HTTP version in the request or status line + * is not `0.9`, `1.0`, `1.1` or `2.0`. + * With this flag the invalid value will be parsed normally. + * + * **Enabling this flag can pose a security issue since you will allow unsupported + * HTTP versions. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_version(llhttp_t* parser, int enabled); + +/* Enables/disables lenient handling of additional data received after a message ends + * and keep-alive is disabled. + * + * Normally `llhttp` would error when additional unexpected data is received if the message + * contains the `Connection` header with `close` value. + * With this flag the extra data will discarded without throwing an error. + * + * **Enabling this flag can pose a security issue since you will be exposed to + * poisoning attacks. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_data_after_close(llhttp_t* parser, int enabled); + +/* Enables/disables lenient handling of incomplete CRLF sequences. + * + * Normally `llhttp` would error when a CR is not followed by LF when terminating the + * request line, the status line, the headers or a chunk header. + * With this flag only a CR is required to terminate such sections. + * + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_optional_lf_after_cr(llhttp_t* parser, int enabled); + +/* + * Enables/disables lenient handling of line separators. + * + * Normally `llhttp` would error when a LF is not preceded by CR when terminating the + * request line, the status line, the headers, a chunk header or a chunk data. + * With this flag only a LF is required to terminate such sections. + * + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_optional_cr_before_lf(llhttp_t* parser, int enabled); + +/* Enables/disables lenient handling of chunks not separated via CRLF. + * + * Normally `llhttp` would error when after a chunk data a CRLF is missing before + * starting a new chunk. + * With this flag the new chunk can start immediately after the previous one. + * + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_optional_crlf_after_chunk(llhttp_t* parser, int enabled); + +/* Enables/disables lenient handling of spaces after chunk size. + * + * Normally `llhttp` would error when after a chunk size is followed by one or more + * spaces are present instead of a CRLF or `;`. + * With this flag this check is disabled. + * + * **Enabling this flag can pose a security issue since you will be exposed to + * request smuggling attacks. USE WITH CAUTION!** + */ +LLHTTP_EXPORT +void llhttp_set_lenient_spaces_after_chunk_size(llhttp_t* parser, int enabled); + #ifdef __cplusplus } /* extern "C" */ #endif #endif /* INCLUDE_LLHTTP_API_H_ */ + #endif /* INCLUDE_LLHTTP_H_ */ diff --git a/src/3rdparty/rapidjson/allocators.h b/src/3rdparty/rapidjson/allocators.h index 8918cf76..c2826e9f 100644 --- a/src/3rdparty/rapidjson/allocators.h +++ b/src/3rdparty/rapidjson/allocators.h @@ -19,6 +19,7 @@ #include "internal/meta.h" #include +#include #if RAPIDJSON_HAS_CXX11 #include @@ -433,7 +434,7 @@ namespace internal { template inline T* Realloc(A& a, T* old_p, size_t old_n, size_t new_n) { - RAPIDJSON_NOEXCEPT_ASSERT(old_n <= SIZE_MAX / sizeof(T) && new_n <= SIZE_MAX / sizeof(T)); + RAPIDJSON_NOEXCEPT_ASSERT(old_n <= (std::numeric_limits::max)() / sizeof(T) && new_n <= (std::numeric_limits::max)() / sizeof(T)); return static_cast(a.Realloc(old_p, old_n * sizeof(T), new_n * sizeof(T))); } @@ -496,9 +497,9 @@ public: #endif /* implicit */ - StdAllocator(const BaseAllocator& allocator) RAPIDJSON_NOEXCEPT : + StdAllocator(const BaseAllocator& baseAllocator) RAPIDJSON_NOEXCEPT : allocator_type(), - baseAllocator_(allocator) + baseAllocator_(baseAllocator) { } ~StdAllocator() RAPIDJSON_NOEXCEPT diff --git a/src/3rdparty/rapidjson/document.h b/src/3rdparty/rapidjson/document.h index 660e14eb..5b1d8579 100644 --- a/src/3rdparty/rapidjson/document.h +++ b/src/3rdparty/rapidjson/document.h @@ -75,7 +75,7 @@ class GenericDocument; User can define this to use CrtAllocator or MemoryPoolAllocator. */ #ifndef RAPIDJSON_DEFAULT_ALLOCATOR -#define RAPIDJSON_DEFAULT_ALLOCATOR MemoryPoolAllocator +#define RAPIDJSON_DEFAULT_ALLOCATOR ::RAPIDJSON_NAMESPACE::MemoryPoolAllocator<::RAPIDJSON_NAMESPACE::CrtAllocator> #endif /*! \def RAPIDJSON_DEFAULT_STACK_ALLOCATOR @@ -85,7 +85,7 @@ class GenericDocument; User can define this to use CrtAllocator or MemoryPoolAllocator. */ #ifndef RAPIDJSON_DEFAULT_STACK_ALLOCATOR -#define RAPIDJSON_DEFAULT_STACK_ALLOCATOR CrtAllocator +#define RAPIDJSON_DEFAULT_STACK_ALLOCATOR ::RAPIDJSON_NAMESPACE::CrtAllocator #endif /*! \def RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY @@ -1033,7 +1033,7 @@ public: return false; for (ConstMemberIterator lhsMemberItr = MemberBegin(); lhsMemberItr != MemberEnd(); ++lhsMemberItr) { typename RhsType::ConstMemberIterator rhsMemberItr = rhs.FindMember(lhsMemberItr->name); - if (rhsMemberItr == rhs.MemberEnd() || lhsMemberItr->value != rhsMemberItr->value) + if (rhsMemberItr == rhs.MemberEnd() || (!(lhsMemberItr->value == rhsMemberItr->value))) return false; } return true; @@ -1042,7 +1042,7 @@ public: if (data_.a.size != rhs.data_.a.size) return false; for (SizeType i = 0; i < data_.a.size; i++) - if ((*this)[i] != rhs[i]) + if (!((*this)[i] == rhs[i])) return false; return true; @@ -1078,6 +1078,7 @@ public: */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); } +#ifndef __cpp_impl_three_way_comparison //! Not-equal-to operator /*! \return !(*this == rhs) */ @@ -1092,7 +1093,6 @@ public: */ template RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); } -#ifndef __cpp_lib_three_way_comparison //! Equal-to operator with arbitrary types (symmetric version) /*! \return (rhs == lhs) */ @@ -1230,13 +1230,28 @@ public: else { RAPIDJSON_ASSERT(false); // see above note - // This will generate -Wexit-time-destructors in clang - // static GenericValue NullValue; - // return NullValue; - - // Use static buffer and placement-new to prevent destruction - static char buffer[sizeof(GenericValue)]; +#if RAPIDJSON_HAS_CXX11 + // Use thread-local storage to prevent races between threads. + // Use static buffer and placement-new to prevent destruction, with + // alignas() to ensure proper alignment. + alignas(GenericValue) thread_local static char buffer[sizeof(GenericValue)]; return *new (buffer) GenericValue(); +#elif defined(_MSC_VER) && _MSC_VER < 1900 + // There's no way to solve both thread locality and proper alignment + // simultaneously. + __declspec(thread) static char buffer[sizeof(GenericValue)]; + return *new (buffer) GenericValue(); +#elif defined(__GNUC__) || defined(__clang__) + // This will generate -Wexit-time-destructors in clang, but that's + // better than having under-alignment. + __thread static GenericValue buffer; + return buffer; +#else + // Don't know what compiler this is, so don't know how to ensure + // thread-locality. + static GenericValue buffer; + return buffer; +#endif } } template @@ -2430,13 +2445,14 @@ private: data_.f.flags = kShortStringFlag; data_.ss.SetLength(s.length); str = data_.ss.str; + std::memmove(str, s, s.length * sizeof(Ch)); } else { data_.f.flags = kCopyStringFlag; data_.s.length = s.length; str = static_cast(allocator.Malloc((s.length + 1) * sizeof(Ch))); SetStringPointer(str); + std::memcpy(str, s, s.length * sizeof(Ch)); } - std::memcpy(str, s, s.length * sizeof(Ch)); str[s.length] = '\0'; } @@ -2486,6 +2502,7 @@ public: typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. typedef GenericValue ValueType; //!< Value type of the document. typedef Allocator AllocatorType; //!< Allocator type from template parameter. + typedef StackAllocator StackAllocatorType; //!< StackAllocator type from template parameter. //! Constructor /*! Creates an empty document of specified type. diff --git a/src/3rdparty/rapidjson/encodings.h b/src/3rdparty/rapidjson/encodings.h index f9f1ed18..a5d88d39 100644 --- a/src/3rdparty/rapidjson/encodings.h +++ b/src/3rdparty/rapidjson/encodings.h @@ -177,10 +177,10 @@ struct UTF8 { template static bool Validate(InputStream& is, OutputStream& os) { -#define RAPIDJSON_COPY() os.Put(c = is.Take()) +#define RAPIDJSON_COPY() if (c != '\0') os.Put(c = is.Take()) #define RAPIDJSON_TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) #define RAPIDJSON_TAIL() RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x70) - Ch c; + Ch c = static_cast(-1); RAPIDJSON_COPY(); if (!(c & 0x80)) return true; diff --git a/src/3rdparty/rapidjson/error/en.h b/src/3rdparty/rapidjson/error/en.h index 7c620a2e..c87b04eb 100644 --- a/src/3rdparty/rapidjson/error/en.h +++ b/src/3rdparty/rapidjson/error/en.h @@ -104,15 +104,69 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetValidateError_En(ValidateErrorCode val case kValidateErrorType: return RAPIDJSON_ERROR_STRING("Property has a type '%actual' that is not in the following list: '%expected'."); case kValidateErrorOneOf: return RAPIDJSON_ERROR_STRING("Property did not match any of the sub-schemas specified by 'oneOf', refer to following errors."); - case kValidateErrorOneOfMatch: return RAPIDJSON_ERROR_STRING("Property matched more than one of the sub-schemas specified by 'oneOf'."); + case kValidateErrorOneOfMatch: return RAPIDJSON_ERROR_STRING("Property matched more than one of the sub-schemas specified by 'oneOf', indices '%matches'."); case kValidateErrorAllOf: return RAPIDJSON_ERROR_STRING("Property did not match all of the sub-schemas specified by 'allOf', refer to following errors."); case kValidateErrorAnyOf: return RAPIDJSON_ERROR_STRING("Property did not match any of the sub-schemas specified by 'anyOf', refer to following errors."); case kValidateErrorNot: return RAPIDJSON_ERROR_STRING("Property matched the sub-schema specified by 'not'."); + case kValidateErrorReadOnly: return RAPIDJSON_ERROR_STRING("Property is read-only but has been provided when validation is for writing."); + case kValidateErrorWriteOnly: return RAPIDJSON_ERROR_STRING("Property is write-only but has been provided when validation is for reading."); + default: return RAPIDJSON_ERROR_STRING("Unknown error."); } } +//! Maps error code of schema document compilation into error message. +/*! + \ingroup RAPIDJSON_ERRORS + \param schemaErrorCode Error code obtained from compiling the schema document. + \return the error message. + \note User can make a copy of this function for localization. + Using switch-case is safer for future modification of error codes. +*/ + inline const RAPIDJSON_ERROR_CHARTYPE* GetSchemaError_En(SchemaErrorCode schemaErrorCode) { + switch (schemaErrorCode) { + case kSchemaErrorNone: return RAPIDJSON_ERROR_STRING("No error."); + + case kSchemaErrorStartUnknown: return RAPIDJSON_ERROR_STRING("Pointer '%value' to start of schema does not resolve to a location in the document."); + case kSchemaErrorRefPlainName: return RAPIDJSON_ERROR_STRING("$ref fragment '%value' must be a JSON pointer."); + case kSchemaErrorRefInvalid: return RAPIDJSON_ERROR_STRING("$ref must not be an empty string."); + case kSchemaErrorRefPointerInvalid: return RAPIDJSON_ERROR_STRING("$ref fragment '%value' is not a valid JSON pointer at offset '%offset'."); + case kSchemaErrorRefUnknown: return RAPIDJSON_ERROR_STRING("$ref '%value' does not resolve to a location in the target document."); + case kSchemaErrorRefCyclical: return RAPIDJSON_ERROR_STRING("$ref '%value' is cyclical."); + case kSchemaErrorRefNoRemoteProvider: return RAPIDJSON_ERROR_STRING("$ref is remote but there is no remote provider."); + case kSchemaErrorRefNoRemoteSchema: return RAPIDJSON_ERROR_STRING("$ref '%value' is remote but the remote provider did not return a schema."); + case kSchemaErrorRegexInvalid: return RAPIDJSON_ERROR_STRING("Invalid regular expression '%value' in 'pattern' or 'patternProperties'."); + case kSchemaErrorSpecUnknown: return RAPIDJSON_ERROR_STRING("JSON schema draft or OpenAPI version is not recognized."); + case kSchemaErrorSpecUnsupported: return RAPIDJSON_ERROR_STRING("JSON schema draft or OpenAPI version is not supported."); + case kSchemaErrorSpecIllegal: return RAPIDJSON_ERROR_STRING("Both JSON schema draft and OpenAPI version found in document."); + case kSchemaErrorReadOnlyAndWriteOnly: return RAPIDJSON_ERROR_STRING("Property must not be both 'readOnly' and 'writeOnly'."); + + default: return RAPIDJSON_ERROR_STRING("Unknown error."); + } + } + +//! Maps error code of pointer parse into error message. +/*! + \ingroup RAPIDJSON_ERRORS + \param pointerParseErrorCode Error code obtained from pointer parse. + \return the error message. + \note User can make a copy of this function for localization. + Using switch-case is safer for future modification of error codes. +*/ +inline const RAPIDJSON_ERROR_CHARTYPE* GetPointerParseError_En(PointerParseErrorCode pointerParseErrorCode) { + switch (pointerParseErrorCode) { + case kPointerParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); + + case kPointerParseErrorTokenMustBeginWithSolidus: return RAPIDJSON_ERROR_STRING("A token must begin with a '/'."); + case kPointerParseErrorInvalidEscape: return RAPIDJSON_ERROR_STRING("Invalid escape."); + case kPointerParseErrorInvalidPercentEncoding: return RAPIDJSON_ERROR_STRING("Invalid percent encoding in URI fragment."); + case kPointerParseErrorCharacterMustPercentEncode: return RAPIDJSON_ERROR_STRING("A character must be percent encoded in a URI fragment."); + + default: return RAPIDJSON_ERROR_STRING("Unknown error."); + } +} + RAPIDJSON_NAMESPACE_END #ifdef __clang__ diff --git a/src/3rdparty/rapidjson/error/error.h b/src/3rdparty/rapidjson/error/error.h index 5c890ba0..cae345db 100644 --- a/src/3rdparty/rapidjson/error/error.h +++ b/src/3rdparty/rapidjson/error/error.h @@ -42,7 +42,7 @@ RAPIDJSON_DIAG_OFF(padded) /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_ERROR_STRING -//! Macro for converting string literial to \ref RAPIDJSON_ERROR_CHARTYPE[]. +//! Macro for converting string literal to \ref RAPIDJSON_ERROR_CHARTYPE[]. /*! \ingroup RAPIDJSON_ERRORS By default this conversion macro does nothing. On Windows, user can define this macro as \c _T(x) for supporting both @@ -185,14 +185,17 @@ enum ValidateErrorCode { kValidateErrorPatternProperties, //!< See other errors. kValidateErrorDependencies, //!< Object has missing property or schema dependencies. - kValidateErrorEnum, //!< Property has a value that is not one of its allowed enumerated values - kValidateErrorType, //!< Property has a type that is not allowed by the schema.. + kValidateErrorEnum, //!< Property has a value that is not one of its allowed enumerated values. + kValidateErrorType, //!< Property has a type that is not allowed by the schema. kValidateErrorOneOf, //!< Property did not match any of the sub-schemas specified by 'oneOf'. kValidateErrorOneOfMatch, //!< Property matched more than one of the sub-schemas specified by 'oneOf'. kValidateErrorAllOf, //!< Property did not match all of the sub-schemas specified by 'allOf'. kValidateErrorAnyOf, //!< Property did not match any of the sub-schemas specified by 'anyOf'. - kValidateErrorNot //!< Property matched the sub-schema specified by 'not'. + kValidateErrorNot, //!< Property matched the sub-schema specified by 'not'. + + kValidateErrorReadOnly, //!< Property is read-only but has been provided when validation is for writing + kValidateErrorWriteOnly //!< Property is write-only but has been provided when validation is for reading }; //! Function pointer type of GetValidateError(). @@ -207,6 +210,72 @@ enum ValidateErrorCode { */ typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetValidateErrorFunc)(ValidateErrorCode); +/////////////////////////////////////////////////////////////////////////////// +// SchemaErrorCode + +//! Error codes when validating. +/*! \ingroup RAPIDJSON_ERRORS + \see GenericSchemaValidator +*/ +enum SchemaErrorCode { + kSchemaErrorNone = 0, //!< No error. + + kSchemaErrorStartUnknown, //!< Pointer to start of schema does not resolve to a location in the document + kSchemaErrorRefPlainName, //!< $ref fragment must be a JSON pointer + kSchemaErrorRefInvalid, //!< $ref must not be an empty string + kSchemaErrorRefPointerInvalid, //!< $ref fragment is not a valid JSON pointer at offset + kSchemaErrorRefUnknown, //!< $ref does not resolve to a location in the target document + kSchemaErrorRefCyclical, //!< $ref is cyclical + kSchemaErrorRefNoRemoteProvider, //!< $ref is remote but there is no remote provider + kSchemaErrorRefNoRemoteSchema, //!< $ref is remote but the remote provider did not return a schema + kSchemaErrorRegexInvalid, //!< Invalid regular expression in 'pattern' or 'patternProperties' + kSchemaErrorSpecUnknown, //!< JSON schema draft or OpenAPI version is not recognized + kSchemaErrorSpecUnsupported, //!< JSON schema draft or OpenAPI version is not supported + kSchemaErrorSpecIllegal, //!< Both JSON schema draft and OpenAPI version found in document + kSchemaErrorReadOnlyAndWriteOnly //!< Property must not be both 'readOnly' and 'writeOnly' +}; + +//! Function pointer type of GetSchemaError(). +/*! \ingroup RAPIDJSON_ERRORS + + This is the prototype for \c GetSchemaError_X(), where \c X is a locale. + User can dynamically change locale in runtime, e.g.: +\code + GetSchemaErrorFunc GetSchemaError = GetSchemaError_En; // or whatever + const RAPIDJSON_ERROR_CHARTYPE* s = GetSchemaError(validator.GetInvalidSchemaCode()); +\endcode +*/ +typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetSchemaErrorFunc)(SchemaErrorCode); + +/////////////////////////////////////////////////////////////////////////////// +// PointerParseErrorCode + +//! Error code of JSON pointer parsing. +/*! \ingroup RAPIDJSON_ERRORS + \see GenericPointer::GenericPointer, GenericPointer::GetParseErrorCode +*/ +enum PointerParseErrorCode { + kPointerParseErrorNone = 0, //!< The parse is successful + + kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' + kPointerParseErrorInvalidEscape, //!< Invalid escape + kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment + kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment +}; + +//! Function pointer type of GetPointerParseError(). +/*! \ingroup RAPIDJSON_ERRORS + + This is the prototype for \c GetPointerParseError_X(), where \c X is a locale. + User can dynamically change locale in runtime, e.g.: +\code + GetPointerParseErrorFunc GetPointerParseError = GetPointerParseError_En; // or whatever + const RAPIDJSON_ERROR_CHARTYPE* s = GetPointerParseError(pointer.GetParseErrorCode()); +\endcode +*/ +typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetPointerParseErrorFunc)(PointerParseErrorCode); + + RAPIDJSON_NAMESPACE_END #ifdef __clang__ diff --git a/src/3rdparty/rapidjson/internal/biginteger.h b/src/3rdparty/rapidjson/internal/biginteger.h index 2e54c7f7..09d0387f 100644 --- a/src/3rdparty/rapidjson/internal/biginteger.h +++ b/src/3rdparty/rapidjson/internal/biginteger.h @@ -19,7 +19,11 @@ #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && defined(_M_AMD64) #include // for _umul128 +#if !defined(_ARM64EC_) #pragma intrinsic(_umul128) +#else +#pragma comment(lib,"softintrin") +#endif #endif RAPIDJSON_NAMESPACE_BEGIN @@ -255,7 +259,7 @@ private: if (low < k) (*outHigh)++; return low; -#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) __extension__ typedef unsigned __int128 uint128; uint128 p = static_cast(a) * static_cast(b); p += k; diff --git a/src/3rdparty/rapidjson/internal/diyfp.h b/src/3rdparty/rapidjson/internal/diyfp.h index a40797ec..1f60fb60 100644 --- a/src/3rdparty/rapidjson/internal/diyfp.h +++ b/src/3rdparty/rapidjson/internal/diyfp.h @@ -25,7 +25,11 @@ #if defined(_MSC_VER) && defined(_M_AMD64) && !defined(__INTEL_COMPILER) #include +#if !defined(_ARM64EC_) #pragma intrinsic(_umul128) +#else +#pragma comment(lib,"softintrin") +#endif #endif RAPIDJSON_NAMESPACE_BEGIN @@ -75,7 +79,7 @@ struct DiyFp { if (l & (uint64_t(1) << 63)) // rounding h++; return DiyFp(h, e + rhs.e + 64); -#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) __extension__ typedef unsigned __int128 uint128; uint128 p = static_cast(f) * static_cast(rhs.f); uint64_t h = static_cast(p >> 64); diff --git a/src/3rdparty/rapidjson/internal/dtoa.h b/src/3rdparty/rapidjson/internal/dtoa.h index aa27f7b6..91c5756d 100644 --- a/src/3rdparty/rapidjson/internal/dtoa.h +++ b/src/3rdparty/rapidjson/internal/dtoa.h @@ -58,11 +58,11 @@ inline int CountDecimalDigit32(uint32_t n) { } inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { - static const uint64_t kPow10[] = { 1U, 10U, 100U, 1000U, 10000U, 100000U, 1000000U, 10000000U, 100000000U, - 1000000000U, 10000000000U, 100000000000U, 1000000000000U, - 10000000000000U, 100000000000000U, 1000000000000000U, - 10000000000000000U, 100000000000000000U, 1000000000000000000U, - 10000000000000000000U }; + static const uint64_t kPow10[] = { 1ULL, 10ULL, 100ULL, 1000ULL, 10000ULL, 100000ULL, 1000000ULL, 10000000ULL, 100000000ULL, + 1000000000ULL, 10000000000ULL, 100000000000ULL, 1000000000000ULL, + 10000000000000ULL, 100000000000000ULL, 1000000000000000ULL, + 10000000000000000ULL, 100000000000000000ULL, 1000000000000000000ULL, + 10000000000000000000ULL }; const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); const DiyFp wp_w = Mp - W; uint32_t p1 = static_cast(Mp.f >> -one.e); diff --git a/src/3rdparty/rapidjson/internal/regex.h b/src/3rdparty/rapidjson/internal/regex.h index e3712610..c412d525 100644 --- a/src/3rdparty/rapidjson/internal/regex.h +++ b/src/3rdparty/rapidjson/internal/regex.h @@ -615,7 +615,7 @@ public: RAPIDJSON_ASSERT(regex_.IsValid()); if (!allocator_) ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); - stateSet_ = static_cast(allocator_->Malloc(GetStateSetSize())); + stateSet_ = static_cast(allocator_->Malloc(GetStateSetSize())); state0_.template Reserve(regex_.stateCount_); state1_.template Reserve(regex_.stateCount_); } diff --git a/src/3rdparty/rapidjson/internal/strtod.h b/src/3rdparty/rapidjson/internal/strtod.h index 62a42c69..3fbd7ae1 100644 --- a/src/3rdparty/rapidjson/internal/strtod.h +++ b/src/3rdparty/rapidjson/internal/strtod.h @@ -134,7 +134,7 @@ inline bool StrtodDiyFp(const Ch* decimals, int dLen, int dExp, double* result) int i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 for (; i < dLen; i++) { if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || - (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > Ch('5'))) + (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] >= Ch('5'))) break; significand = significand * 10u + static_cast(decimals[i] - Ch('0')); } diff --git a/src/3rdparty/rapidjson/pointer.h b/src/3rdparty/rapidjson/pointer.h index d89f943c..32fbf88e 100644 --- a/src/3rdparty/rapidjson/pointer.h +++ b/src/3rdparty/rapidjson/pointer.h @@ -18,6 +18,7 @@ #include "document.h" #include "uri.h" #include "internal/itoa.h" +#include "error/error.h" // PointerParseErrorCode #ifdef __clang__ RAPIDJSON_DIAG_PUSH @@ -27,23 +28,16 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated #endif +#if defined(RAPIDJSON_CPLUSPLUS) && RAPIDJSON_CPLUSPLUS >= 201703L +#define RAPIDJSON_IF_CONSTEXPR if constexpr +#else +#define RAPIDJSON_IF_CONSTEXPR if +#endif + RAPIDJSON_NAMESPACE_BEGIN static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token -//! Error code of parsing. -/*! \ingroup RAPIDJSON_ERRORS - \see GenericPointer::GenericPointer, GenericPointer::GetParseErrorCode -*/ -enum PointerParseErrorCode { - kPointerParseErrorNone = 0, //!< The parse is successful - - kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' - kPointerParseErrorInvalidEscape, //!< Invalid escape - kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment - kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment -}; - /////////////////////////////////////////////////////////////////////////////// // GenericPointer @@ -84,7 +78,7 @@ public: typedef GenericUri UriType; - //! A token is the basic units of internal representation. + //! A token is the basic units of internal representation. /*! A JSON pointer string representation "/foo/123" is parsed to two tokens: "foo" and 123. 123 will be represented in both numeric form and string form. @@ -303,7 +297,7 @@ public: SizeType length = static_cast(end - buffer); buffer[length] = '\0'; - if (sizeof(Ch) == 1) { + RAPIDJSON_IF_CONSTEXPR (sizeof(Ch) == 1) { Token token = { reinterpret_cast(buffer), length, index }; return Append(token, allocator); } @@ -902,10 +896,16 @@ private: std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); } - // Adjust pointers to name buffer - std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; - for (Token *t = tokens_; t != tokens_ + rhs.tokenCount_; ++t) - t->name += diff; + // The names of each token point to a string in the nameBuffer_. The + // previous memcpy copied over string pointers into the rhs.nameBuffer_, + // but they should point to the strings in the new nameBuffer_. + for (size_t i = 0; i < rhs.tokenCount_; ++i) { + // The offset between the string address and the name buffer should + // still be constant, so we can just get this offset and set each new + // token name according the new buffer start + the known offset. + std::ptrdiff_t name_offset = rhs.tokens_[i].name - rhs.nameBuffer_; + tokens_[i].name = nameBuffer_ + name_offset; + } return nameBuffer_ + nameBufferSize; } diff --git a/src/3rdparty/rapidjson/rapidjson.h b/src/3rdparty/rapidjson/rapidjson.h index d5f83f70..fa7db400 100644 --- a/src/3rdparty/rapidjson/rapidjson.h +++ b/src/3rdparty/rapidjson/rapidjson.h @@ -195,7 +195,7 @@ */ #ifndef RAPIDJSON_NO_INT64DEFINE //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 +#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 #include "msinttypes/stdint.h" #include "msinttypes/inttypes.h" #else @@ -268,7 +268,7 @@ # elif defined(_BIG_ENDIAN) && !defined(_LITTLE_ENDIAN) # define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN // Detect with architecture macros -# elif defined(__sparc) || defined(__sparc__) || defined(_POWER) || defined(__powerpc__) || defined(__ppc__) || defined(__hpux) || defined(__hppa) || defined(_MIPSEB) || defined(_POWER) || defined(__s390__) +# elif defined(__sparc) || defined(__sparc__) || defined(_POWER) || defined(__powerpc__) || defined(__ppc__) || defined(__ppc64__) || defined(__hpux) || defined(__hppa) || defined(_MIPSEB) || defined(_POWER) || defined(__s390__) # define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN # elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || defined(__bfin__) # define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN diff --git a/src/3rdparty/rapidjson/reader.h b/src/3rdparty/rapidjson/reader.h index 542b7c00..f7ef6102 100644 --- a/src/3rdparty/rapidjson/reader.h +++ b/src/3rdparty/rapidjson/reader.h @@ -1433,7 +1433,7 @@ private: class NumberStream : public NumberStream { typedef NumberStream Base; public: - NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} + NumberStream(GenericReader& reader, InputStream& s) : Base(reader, s), stackStream(reader.stack_) {} RAPIDJSON_FORCEINLINE Ch TakePush() { stackStream.Put(static_cast(Base::is.Peek())); @@ -1459,7 +1459,7 @@ private: class NumberStream : public NumberStream { typedef NumberStream Base; public: - NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} + NumberStream(GenericReader& reader, InputStream& s) : Base(reader, s) {} RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } }; @@ -1584,7 +1584,7 @@ private: // Parse frac = decimal-point 1*DIGIT int expFrac = 0; size_t decimalPosition; - if (Consume(s, '.')) { + if (!useNanOrInf && Consume(s, '.')) { decimalPosition = s.Length(); if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9'))) @@ -1631,7 +1631,7 @@ private: // Parse exp = e [ minus / plus ] 1*DIGIT int exp = 0; - if (Consume(s, 'e') || Consume(s, 'E')) { + if (!useNanOrInf && (Consume(s, 'e') || Consume(s, 'E'))) { if (!useDouble) { d = static_cast(use64bit ? i64 : i); useDouble = true; @@ -1694,7 +1694,7 @@ private: } else { SizeType numCharsToCopy = static_cast(s.Length()); - GenericStringStream> srcStream(s.Pop()); + GenericStringStream > srcStream(s.Pop()); StackStream dstStream(stack_); while (numCharsToCopy--) { Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); diff --git a/src/3rdparty/rapidjson/schema.h b/src/3rdparty/rapidjson/schema.h index 1db4b57c..89875d14 100644 --- a/src/3rdparty/rapidjson/schema.h +++ b/src/3rdparty/rapidjson/schema.h @@ -24,13 +24,9 @@ #if !defined(RAPIDJSON_SCHEMA_USE_INTERNALREGEX) #define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 1 -#else -#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 0 #endif -#if !RAPIDJSON_SCHEMA_USE_INTERNALREGEX && defined(RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) -#define RAPIDJSON_SCHEMA_USE_STDREGEX 1 -#else +#if !defined(RAPIDJSON_SCHEMA_USE_STDREGEX) || !(__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) #define RAPIDJSON_SCHEMA_USE_STDREGEX 0 #endif @@ -50,10 +46,6 @@ #define RAPIDJSON_SCHEMA_VERBOSE 0 #endif -#if RAPIDJSON_SCHEMA_VERBOSE -#include "stringbuffer.h" -#endif - RAPIDJSON_DIAG_PUSH #if defined(__GNUC__) @@ -78,48 +70,94 @@ RAPIDJSON_NAMESPACE_BEGIN namespace internal { -inline void PrintInvalidKeyword(const char* keyword) { - printf("Fail keyword: %s\n", keyword); +inline void PrintInvalidKeywordData(const char* keyword) { + printf(" Fail keyword: '%s'\n", keyword); } -inline void PrintInvalidKeyword(const wchar_t* keyword) { - wprintf(L"Fail keyword: %ls\n", keyword); +inline void PrintInvalidKeywordData(const wchar_t* keyword) { + wprintf(L" Fail keyword: '%ls'\n", keyword); } -inline void PrintInvalidDocument(const char* document) { - printf("Fail document: %s\n\n", document); +inline void PrintInvalidDocumentData(const char* document) { + printf(" Fail document: '%s'\n", document); } -inline void PrintInvalidDocument(const wchar_t* document) { - wprintf(L"Fail document: %ls\n\n", document); +inline void PrintInvalidDocumentData(const wchar_t* document) { + wprintf(L" Fail document: '%ls'\n", document); } -inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) { - printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); +inline void PrintValidatorPointersData(const char* s, const char* d, unsigned depth) { + printf(" Sch: %*s'%s'\n Doc: %*s'%s'\n", depth * 4, " ", s, depth * 4, " ", d); } -inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) { - wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); +inline void PrintValidatorPointersData(const wchar_t* s, const wchar_t* d, unsigned depth) { + wprintf(L" Sch: %*ls'%ls'\n Doc: %*ls'%ls'\n", depth * 4, L" ", s, depth * 4, L" ", d); +} + +inline void PrintSchemaIdsData(const char* base, const char* local, const char* resolved) { + printf(" Resolving id: Base: '%s', Local: '%s', Resolved: '%s'\n", base, local, resolved); +} + +inline void PrintSchemaIdsData(const wchar_t* base, const wchar_t* local, const wchar_t* resolved) { + wprintf(L" Resolving id: Base: '%ls', Local: '%ls', Resolved: '%ls'\n", base, local, resolved); +} + +inline void PrintMethodData(const char* method) { + printf("%s\n", method); +} + +inline void PrintMethodData(const char* method, bool b) { + printf("%s, Data: '%s'\n", method, b ? "true" : "false"); +} + +inline void PrintMethodData(const char* method, int64_t i) { + printf("%s, Data: '%" PRId64 "'\n", method, i); +} + +inline void PrintMethodData(const char* method, uint64_t u) { + printf("%s, Data: '%" PRIu64 "'\n", method, u); +} + +inline void PrintMethodData(const char* method, double d) { + printf("%s, Data: '%lf'\n", method, d); +} + +inline void PrintMethodData(const char* method, const char* s) { + printf("%s, Data: '%s'\n", method, s); +} + +inline void PrintMethodData(const char* method, const wchar_t* s) { + wprintf(L"%hs, Data: '%ls'\n", method, s); +} + +inline void PrintMethodData(const char* method, const char* s1, const char* s2) { + printf("%s, Data: '%s', '%s'\n", method, s1, s2); +} + +inline void PrintMethodData(const char* method, const wchar_t* s1, const wchar_t* s2) { + wprintf(L"%hs, Data: '%ls', '%ls'\n", method, s1, s2); } } // namespace internal #endif // RAPIDJSON_SCHEMA_VERBOSE +#ifndef RAPIDJSON_SCHEMA_PRINT +#if RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_SCHEMA_PRINT(name, ...) internal::Print##name##Data(__VA_ARGS__) +#else +#define RAPIDJSON_SCHEMA_PRINT(name, ...) +#endif +#endif + /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_INVALID_KEYWORD_RETURN -#if RAPIDJSON_SCHEMA_VERBOSE -#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) internal::PrintInvalidKeyword(keyword) -#else -#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) -#endif - #define RAPIDJSON_INVALID_KEYWORD_RETURN(code)\ RAPIDJSON_MULTILINEMACRO_BEGIN\ context.invalidCode = code;\ context.invalidKeyword = SchemaType::GetValidateErrorKeyword(code).GetString();\ - RAPIDJSON_INVALID_KEYWORD_VERBOSE(context.invalidKeyword);\ + RAPIDJSON_SCHEMA_PRINT(InvalidKeyword, context.invalidKeyword);\ return false;\ RAPIDJSON_MULTILINEMACRO_END @@ -137,14 +175,56 @@ RAPIDJSON_MULTILINEMACRO_END #endif //! Combination of validate flags -/*! \see - */ enum ValidateFlag { kValidateNoFlags = 0, //!< No flags are set. kValidateContinueOnErrorFlag = 1, //!< Don't stop after first validation error. + kValidateReadFlag = 2, //!< Validation is for a read semantic. + kValidateWriteFlag = 4, //!< Validation is for a write semantic. kValidateDefaultFlags = RAPIDJSON_VALIDATE_DEFAULT_FLAGS //!< Default validate flags. Can be customized by defining RAPIDJSON_VALIDATE_DEFAULT_FLAGS }; +/////////////////////////////////////////////////////////////////////////////// +// Specification +enum SchemaDraft { + kDraftUnknown = -1, + kDraftNone = 0, + kDraft03 = 3, + kDraftMin = 4, //!< Current minimum supported draft + kDraft04 = 4, + kDraft05 = 5, + kDraftMax = 5, //!< Current maximum supported draft + kDraft06 = 6, + kDraft07 = 7, + kDraft2019_09 = 8, + kDraft2020_12 = 9 +}; + +enum OpenApiVersion { + kVersionUnknown = -1, + kVersionNone = 0, + kVersionMin = 2, //!< Current minimum supported version + kVersion20 = 2, + kVersion30 = 3, + kVersionMax = 3, //!< Current maximum supported version + kVersion31 = 4, +}; + +struct Specification { + Specification(SchemaDraft d) : draft(d), oapi(kVersionNone) {} + Specification(OpenApiVersion o) : oapi(o) { + if (oapi == kVersion20) draft = kDraft04; + else if (oapi == kVersion30) draft = kDraft05; + else if (oapi == kVersion31) draft = kDraft2020_12; + else draft = kDraft04; + } + ~Specification() {} + bool IsSupported() const { + return ((draft >= kDraftMin && draft <= kDraftMax) && ((oapi == kVersionNone) || (oapi >= kVersionMin && oapi <= kVersionMax))); + } + SchemaDraft draft; + OpenApiVersion oapi; +}; + /////////////////////////////////////////////////////////////////////////////// // Forward declarations @@ -234,8 +314,11 @@ public: virtual void EndDisallowedType(const typename SchemaType::ValueType& actualType) = 0; virtual void NotAllOf(ISchemaValidator** subvalidators, SizeType count) = 0; virtual void NoneOf(ISchemaValidator** subvalidators, SizeType count) = 0; - virtual void NotOneOf(ISchemaValidator** subvalidators, SizeType count, bool matched) = 0; + virtual void NotOneOf(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void MultipleOneOf(SizeType index1, SizeType index2) = 0; virtual void Disallowed() = 0; + virtual void DisallowedWhenWriting() = 0; + virtual void DisallowedWhenReading() = 0; }; @@ -280,7 +363,9 @@ public: uint64_t h = Hash(0, kObjectType); uint64_t* kv = stack_.template Pop(memberCount * 2); for (SizeType i = 0; i < memberCount; i++) - h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive + // Issue #2205 + // Hasing the key to avoid key=value cases with bug-prone zero-value hash + h ^= Hash(Hash(0, kv[i * 2]), kv[i * 2 + 1]); // Use xor to achieve member order insensitive *stack_.template Push() = h; return true; } @@ -318,7 +403,7 @@ private: bool WriteBuffer(Type type, const void* data, size_t len) { // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ - uint64_t h = Hash(RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); + uint64_t h = Hash(RAPIDJSON_UINT64_C2(0xcbf29ce4, 0x84222325), type); const unsigned char* d = static_cast(data); for (size_t i = 0; i < len; i++) h = Hash(h, d[i]); @@ -353,10 +438,11 @@ struct SchemaValidationContext { kPatternValidatorWithAdditionalProperty }; - SchemaValidationContext(SchemaValidatorFactoryType& f, ErrorHandlerType& eh, const SchemaType* s) : + SchemaValidationContext(SchemaValidatorFactoryType& f, ErrorHandlerType& eh, const SchemaType* s, unsigned fl = 0) : factory(f), error_handler(eh), schema(s), + flags(fl), valueSchema(), invalidKeyword(), invalidCode(), @@ -380,13 +466,19 @@ struct SchemaValidationContext { if (hasher) factory.DestroryHasher(hasher); if (validators) { - for (SizeType i = 0; i < validatorCount; i++) - factory.DestroySchemaValidator(validators[i]); + for (SizeType i = 0; i < validatorCount; i++) { + if (validators[i]) { + factory.DestroySchemaValidator(validators[i]); + } + } factory.FreeState(validators); } if (patternPropertiesValidators) { - for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) - factory.DestroySchemaValidator(patternPropertiesValidators[i]); + for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) { + if (patternPropertiesValidators[i]) { + factory.DestroySchemaValidator(patternPropertiesValidators[i]); + } + } factory.FreeState(patternPropertiesValidators); } if (patternPropertiesSchemas) @@ -398,6 +490,7 @@ struct SchemaValidationContext { SchemaValidatorFactoryType& factory; ErrorHandlerType& error_handler; const SchemaType* schema; + unsigned flags; const SchemaType* valueSchema; const Ch* invalidKeyword; ValidateErrorCode invalidCode; @@ -439,7 +532,8 @@ public: Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator, const UriType& id = UriType()) : allocator_(allocator), uri_(schemaDocument->GetURI(), *allocator), - id_(id), + id_(id, allocator), + spec_(schemaDocument->GetSpecification()), pointer_(p, allocator), typeless_(schemaDocument->GetTypeless()), enum_(), @@ -472,8 +566,15 @@ public: maxLength_(~SizeType(0)), exclusiveMinimum_(false), exclusiveMaximum_(false), - defaultValueLength_(0) + defaultValueLength_(0), + readOnly_(false), + writeOnly_(false), + nullable_(false) { + GenericStringBuffer sb; + p.StringifyUriFragment(sb); + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::Schema", sb.GetString(), id.GetString()); + typedef typename ValueType::ConstValueIterator ConstValueIterator; typedef typename ValueType::ConstMemberIterator ConstMemberIterator; @@ -492,10 +593,13 @@ public: return; // If we have an id property, resolve it with the in-scope id + // Not supported for open api 2.0 or 3.0 + if (spec_.oapi != kVersion20 && spec_.oapi != kVersion30) if (const ValueType* v = GetMember(value, GetIdString())) { if (v->IsString()) { UriType local(*v, allocator); id_ = local.Resolve(id_, allocator); + RAPIDJSON_SCHEMA_PRINT(SchemaIds, id.GetString(), v->GetString(), id_.GetString()); } } @@ -512,9 +616,9 @@ public: if (v->IsArray() && v->Size() > 0) { enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { - typedef Hasher > EnumHasherType; + typedef Hasher > EnumHasherType; char buffer[256u + 24]; - MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); + MemoryPoolAllocator hasherAllocator(buffer, sizeof(buffer)); EnumHasherType h(&hasherAllocator, 256); itr->Accept(h); enum_[enumCount_++] = h.GetHashCode(); @@ -522,8 +626,11 @@ public: } } - if (schemaDocument) { + if (schemaDocument) AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); + + // AnyOf, OneOf, Not not supported for open api 2.0 + if (schemaDocument && spec_.oapi != kVersion20) { AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); @@ -552,6 +659,8 @@ public: if (itr->IsString()) AddUniqueElement(allProperties, *itr); + // Dependencies not supported for open api 2.0 and 3.0 + if (spec_.oapi != kVersion20 && spec_.oapi != kVersion30) if (dependencies && dependencies->IsObject()) for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { AddUniqueElement(allProperties, itr->name); @@ -581,6 +690,8 @@ public: } } + // PatternProperties not supported for open api 2.0 and 3.0 + if (spec_.oapi != kVersion20 && spec_.oapi != kVersion30) if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) { PointerType q = p.Append(GetPatternPropertiesString(), allocator_); patternProperties_ = static_cast(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); @@ -588,8 +699,9 @@ public: for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) { new (&patternProperties_[patternPropertyCount_]) PatternProperty(); - patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); - schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document, id_); + PointerType r = q.Append(itr->name, allocator_); + patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name, schemaDocument, r); + schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, r, itr->value, document, id_); patternPropertyCount_++; } } @@ -604,6 +716,8 @@ public: } } + // Dependencies not supported for open api 2.0 and 3.0 + if (spec_.oapi != kVersion20 && spec_.oapi != kVersion30) if (dependencies && dependencies->IsObject()) { PointerType q = p.Append(GetDependenciesString(), allocator_); hasDependencies_ = true; @@ -655,6 +769,8 @@ public: AssignIfExist(minItems_, value, GetMinItemsString()); AssignIfExist(maxItems_, value, GetMaxItemsString()); + // AdditionalItems not supported for openapi 2.0 and 3.0 + if (spec_.oapi != kVersion20 && spec_.oapi != kVersion30) if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) { if (v->IsBool()) additionalItems_ = v->GetBool(); @@ -669,7 +785,7 @@ public: AssignIfExist(maxLength_, value, GetMaxLengthString()); if (const ValueType* v = GetMember(value, GetPatternString())) - pattern_ = CreatePattern(*v); + pattern_ = CreatePattern(*v, schemaDocument, p.Append(GetPatternString(), allocator_)); // Number if (const ValueType* v = GetMember(value, GetMinimumString())) @@ -692,6 +808,23 @@ public: if (v->IsString()) defaultValueLength_ = v->GetStringLength(); + // ReadOnly - open api only (until draft 7 supported) + // WriteOnly - open api 3 only (until draft 7 supported) + // Both can't be true + if (spec_.oapi != kVersionNone) + AssignIfExist(readOnly_, value, GetReadOnlyString()); + if (spec_.oapi >= kVersion30) + AssignIfExist(writeOnly_, value, GetWriteOnlyString()); + if (readOnly_ && writeOnly_) + schemaDocument->SchemaError(kSchemaErrorReadOnlyAndWriteOnly, p); + + // Nullable - open api 3 only + // If true add 'null' as allowable type + if (spec_.oapi >= kVersion30) { + AssignIfExist(nullable_, value, GetNullableString()); + if (nullable_) + AddType(GetNullString()); + } } ~Schema() { @@ -723,11 +856,16 @@ public: return id_; } + const Specification& GetSpecification() const { + return spec_; + } + const PointerType& GetPointer() const { return pointer_; } bool BeginValue(Context& context) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::BeginValue"); if (context.inArray) { if (uniqueItems_) context.valueUniqueness = true; @@ -759,6 +897,7 @@ public: } RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::EndValue"); // Only check pattern properties if we have validators if (context.patternPropertiesValidatorCount > 0) { bool otherValid = false; @@ -822,16 +961,19 @@ public: if (oneOf_.schemas) { bool oneValid = false; + SizeType firstMatch = 0; for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) if (context.validators[i]->IsValid()) { if (oneValid) { - context.error_handler.NotOneOf(&context.validators[oneOf_.begin], oneOf_.count, true); + context.error_handler.MultipleOneOf(firstMatch, i - oneOf_.begin); RAPIDJSON_INVALID_KEYWORD_RETURN(kValidateErrorOneOfMatch); - } else + } else { oneValid = true; + firstMatch = i - oneOf_.begin; + } } if (!oneValid) { - context.error_handler.NotOneOf(&context.validators[oneOf_.begin], oneOf_.count, false); + context.error_handler.NotOneOf(&context.validators[oneOf_.begin], oneOf_.count); RAPIDJSON_INVALID_KEYWORD_RETURN(kValidateErrorOneOf); } } @@ -846,6 +988,7 @@ public: } bool Null(Context& context) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::Null"); if (!(type_ & (1 << kNullSchemaType))) { DisallowedType(context, GetNullString()); RAPIDJSON_INVALID_KEYWORD_RETURN(kValidateErrorType); @@ -853,39 +996,43 @@ public: return CreateParallelValidator(context); } - bool Bool(Context& context, bool) const { - if (!(type_ & (1 << kBooleanSchemaType))) { - DisallowedType(context, GetBooleanString()); - RAPIDJSON_INVALID_KEYWORD_RETURN(kValidateErrorType); - } + bool Bool(Context& context, bool b) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::Bool", b); + if (!CheckBool(context, b)) + return false; return CreateParallelValidator(context); } bool Int(Context& context, int i) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::Int", (int64_t)i); if (!CheckInt(context, i)) return false; return CreateParallelValidator(context); } bool Uint(Context& context, unsigned u) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::Uint", (uint64_t)u); if (!CheckUint(context, u)) return false; return CreateParallelValidator(context); } bool Int64(Context& context, int64_t i) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::Int64", i); if (!CheckInt(context, i)) return false; return CreateParallelValidator(context); } bool Uint64(Context& context, uint64_t u) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::Uint64", u); if (!CheckUint(context, u)) return false; return CreateParallelValidator(context); } bool Double(Context& context, double d) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::Double", d); if (!(type_ & (1 << kNumberSchemaType))) { DisallowedType(context, GetNumberString()); RAPIDJSON_INVALID_KEYWORD_RETURN(kValidateErrorType); @@ -904,6 +1051,7 @@ public: } bool String(Context& context, const Ch* str, SizeType length, bool) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::String", str); if (!(type_ & (1 << kStringSchemaType))) { DisallowedType(context, GetStringString()); RAPIDJSON_INVALID_KEYWORD_RETURN(kValidateErrorType); @@ -932,6 +1080,7 @@ public: } bool StartObject(Context& context) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::StartObject"); if (!(type_ & (1 << kObjectSchemaType))) { DisallowedType(context, GetObjectString()); RAPIDJSON_INVALID_KEYWORD_RETURN(kValidateErrorType); @@ -953,6 +1102,8 @@ public: } bool Key(Context& context, const Ch* str, SizeType len, bool) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::Key", str); + if (patternProperties_) { context.patternPropertiesSchemaCount = 0; for (SizeType i = 0; i < patternPropertyCount_; i++) @@ -1004,6 +1155,7 @@ public: } bool EndObject(Context& context, SizeType memberCount) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::EndObject"); if (hasRequired_) { context.error_handler.StartMissingProperties(); for (SizeType index = 0; index < propertyCount_; index++) @@ -1051,6 +1203,7 @@ public: } bool StartArray(Context& context) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::StartArray"); context.arrayElementIndex = 0; context.inArray = true; // Ensure we note that we are in an array @@ -1063,6 +1216,7 @@ public: } bool EndArray(Context& context, SizeType elementCount) const { + RAPIDJSON_SCHEMA_PRINT(Method, "Schema::EndArray"); context.inArray = false; if (elementCount < minItems_) { @@ -1111,6 +1265,9 @@ public: case kValidateErrorAnyOf: return GetAnyOfString(); case kValidateErrorNot: return GetNotString(); + case kValidateErrorReadOnly: return GetReadOnlyString(); + case kValidateErrorWriteOnly: return GetWriteOnlyString(); + default: return GetNullString(); } } @@ -1158,15 +1315,14 @@ public: RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') RAPIDJSON_STRING_(DefaultValue, 'd', 'e', 'f', 'a', 'u', 'l', 't') + RAPIDJSON_STRING_(Schema, '$', 's', 'c', 'h', 'e', 'm', 'a') RAPIDJSON_STRING_(Ref, '$', 'r', 'e', 'f') RAPIDJSON_STRING_(Id, 'i', 'd') - - RAPIDJSON_STRING_(SchemeEnd, ':') - RAPIDJSON_STRING_(AuthStart, '/', '/') - RAPIDJSON_STRING_(QueryStart, '?') - RAPIDJSON_STRING_(FragStart, '#') - RAPIDJSON_STRING_(Slash, '/') - RAPIDJSON_STRING_(Dot, '.') + RAPIDJSON_STRING_(Swagger, 's', 'w', 'a', 'g', 'g', 'e', 'r') + RAPIDJSON_STRING_(OpenApi, 'o', 'p', 'e', 'n', 'a', 'p', 'i') + RAPIDJSON_STRING_(ReadOnly, 'r', 'e', 'a', 'd', 'O', 'n', 'l', 'y') + RAPIDJSON_STRING_(WriteOnly, 'w', 'r', 'i', 't', 'e', 'O', 'n', 'l', 'y') + RAPIDJSON_STRING_(Nullable, 'n', 'u', 'l', 'l', 'a', 'b', 'l', 'e') #undef RAPIDJSON_STRING_ @@ -1241,10 +1397,11 @@ private: #if RAPIDJSON_SCHEMA_USE_INTERNALREGEX template - RegexType* CreatePattern(const ValueType& value) { + RegexType* CreatePattern(const ValueType& value, SchemaDocumentType* sd, const PointerType& p) { if (value.IsString()) { RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), allocator_); if (!r->IsValid()) { + sd->SchemaErrorValue(kSchemaErrorRegexInvalid, p, value.GetString(), value.GetStringLength()); r->~RegexType(); AllocatorType::Free(r); r = 0; @@ -1260,13 +1417,14 @@ private: } #elif RAPIDJSON_SCHEMA_USE_STDREGEX template - RegexType* CreatePattern(const ValueType& value) { + RegexType* CreatePattern(const ValueType& value, SchemaDocumentType* sd, const PointerType& p) { if (value.IsString()) { RegexType *r = static_cast(allocator_->Malloc(sizeof(RegexType))); try { return new (r) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); } - catch (const std::regex_error&) { + catch (const std::regex_error& e) { + sd->SchemaErrorValue(kSchemaErrorRegexInvalid, p, value.GetString(), value.GetStringLength()); AllocatorType::Free(r); } } @@ -1279,7 +1437,9 @@ private: } #else template - RegexType* CreatePattern(const ValueType&) { return 0; } + RegexType* CreatePattern(const ValueType&) { + return 0; + } static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } #endif // RAPIDJSON_SCHEMA_USE_STDREGEX @@ -1294,6 +1454,9 @@ private: else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); } + // Creates parallel validators for allOf, anyOf, oneOf, not and schema dependencies, if required. + // Also creates a hasher for enums and array uniqueness, if required. + // Also a useful place to add type-independent error checks. bool CreateParallelValidator(Context& context) const { if (enum_ || context.arrayUniqueness) context.hasher = context.factory.CreateHasher(); @@ -1301,6 +1464,7 @@ private: if (validatorCount_) { RAPIDJSON_ASSERT(context.validators == 0); context.validators = static_cast(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); + std::memset(context.validators, 0, sizeof(ISchemaValidator*) * validatorCount_); context.validatorCount = validatorCount_; // Always return after first failure for these sub-validators @@ -1323,6 +1487,16 @@ private: } } + // Add any other type-independent checks here + if (readOnly_ && (context.flags & kValidateWriteFlag)) { + context.error_handler.DisallowedWhenWriting(); + RAPIDJSON_INVALID_KEYWORD_RETURN(kValidateErrorReadOnly); + } + if (writeOnly_ && (context.flags & kValidateReadFlag)) { + context.error_handler.DisallowedWhenReading(); + RAPIDJSON_INVALID_KEYWORD_RETURN(kValidateErrorWriteOnly); + } + return true; } @@ -1345,6 +1519,14 @@ private: return false; } + bool CheckBool(Context& context, bool) const { + if (!(type_ & (1 << kBooleanSchemaType))) { + DisallowedType(context, GetBooleanString()); + RAPIDJSON_INVALID_KEYWORD_RETURN(kValidateErrorType); + } + return true; + } + bool CheckInt(Context& context, int64_t i) const { if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) { DisallowedType(context, GetIntegerString()); @@ -1459,9 +1641,12 @@ private: bool CheckDoubleMultipleOf(Context& context, double d) const { double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); - double q = std::floor(a / b); - double r = a - q * b; - if (r > 0.0) { + double q = a / b; + double qRounded = std::floor(q + 0.5); + double scaledEpsilon = (q + qRounded) * std::numeric_limits::epsilon(); + double difference = std::abs(qRounded - q); + bool isMultiple = difference <= scaledEpsilon || difference < (std::numeric_limits::min)(); + if (!isMultiple) { context.error_handler.NotMultipleOf(d, multipleOf_); RAPIDJSON_INVALID_KEYWORD_RETURN(kValidateErrorMultipleOf); } @@ -1510,6 +1695,7 @@ private: AllocatorType* allocator_; SValue uri_; UriType id_; + Specification spec_; PointerType pointer_; const SchemaType* typeless_; uint64_t* enum_; @@ -1554,6 +1740,10 @@ private: bool exclusiveMaximum_; SizeType defaultValueLength_; + + bool readOnly_; + bool writeOnly_; + bool nullable_; }; template @@ -1571,7 +1761,7 @@ struct TokenHelper { template struct TokenHelper { RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { - if (sizeof(SizeType) == 4) { + RAPIDJSON_IF_CONSTEXPR (sizeof(SizeType) == 4) { char *buffer = documentStack.template Push(1 + 10); // '/' + uint *buffer++ = '/'; const char* end = internal::u32toa(index, buffer); @@ -1600,7 +1790,13 @@ public: virtual ~IGenericRemoteSchemaDocumentProvider() {} virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; - virtual const SchemaDocumentType* GetRemoteDocument(GenericUri uri) { return GetRemoteDocument(uri.GetBaseString(), uri.GetBaseStringLength()); } + virtual const SchemaDocumentType* GetRemoteDocument(const GenericUri uri, Specification& spec) { + // Default implementation just calls through for compatibility + // Following line suppresses unused parameter warning + (void)spec; + // printf("GetRemoteDocument: %d %d\n", spec.draft, spec.oapi); + return GetRemoteDocument(uri.GetBaseString(), uri.GetBaseStringLength()); + } }; /////////////////////////////////////////////////////////////////////////////// @@ -1625,8 +1821,9 @@ public: typedef typename EncodingType::Ch Ch; typedef internal::Schema SchemaType; typedef GenericPointer PointerType; - typedef GenericValue SValue; + typedef GenericValue GValue; typedef GenericUri UriType; + typedef GenericStringRef StringRefType; friend class internal::Schema; template friend class GenericSchemaValidator; @@ -1641,18 +1838,24 @@ public: \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. \param allocator An optional allocator instance for allocating memory. Can be null. \param pointer An optional JSON pointer to the start of the schema document + \param spec Optional schema draft or OpenAPI version. Used if no specification in document. Defaults to draft-04. */ explicit GenericSchemaDocument(const ValueType& document, const Ch* uri = 0, SizeType uriLength = 0, IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0, - const PointerType& pointer = PointerType()) : // PR #1393 + const PointerType& pointer = PointerType(), // PR #1393 + const Specification& spec = Specification(kDraft04)) : remoteProvider_(remoteProvider), allocator_(allocator), ownAllocator_(), root_(), typeless_(), schemaMap_(allocator, kInitialSchemaMapSize), - schemaRef_(allocator, kInitialSchemaRefSize) + schemaRef_(allocator, kInitialSchemaRefSize), + spec_(spec), + error_(kObjectType), + currentError_() { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaDocument::GenericSchemaDocument"); if (!allocator_) ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); @@ -1663,6 +1866,10 @@ public: typeless_ = static_cast(allocator_->Malloc(sizeof(SchemaType))); new (typeless_) SchemaType(this, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), allocator_, docId_); + // Establish the schema draft or open api version. + // We only ever look for '$schema' or 'swagger' or 'openapi' at the root of the document. + SetSchemaSpecification(document); + // Generate root schema, it will call CreateSchema() to create sub-schemas, // And call HandleRefSchema() if there are $ref. // PR #1393 use input pointer if supplied @@ -1673,6 +1880,11 @@ public: else if (const ValueType* v = pointer.Get(document)) { CreateSchema(&root_, pointer, *v, document, docId_); } + else { + GenericStringBuffer sb; + pointer.StringifyUriFragment(sb); + SchemaErrorValue(kSchemaErrorStartUnknown, PointerType(), sb.GetString(), static_cast(sb.GetSize() / sizeof(Ch))); + } RAPIDJSON_ASSERT(root_ != 0); @@ -1690,7 +1902,10 @@ public: schemaMap_(std::move(rhs.schemaMap_)), schemaRef_(std::move(rhs.schemaRef_)), uri_(std::move(rhs.uri_)), - docId_(rhs.docId_) + docId_(std::move(rhs.docId_)), + spec_(rhs.spec_), + error_(std::move(rhs.error_)), + currentError_(std::move(rhs.currentError_)) { rhs.remoteProvider_ = 0; rhs.allocator_ = 0; @@ -1709,15 +1924,81 @@ public: Allocator::Free(typeless_); } + // these may contain some allocator data so clear before deleting ownAllocator_ + uri_.SetNull(); + error_.SetNull(); + currentError_.SetNull(); + RAPIDJSON_DELETE(ownAllocator_); } - const SValue& GetURI() const { return uri_; } + const GValue& GetURI() const { return uri_; } + + const Specification& GetSpecification() const { return spec_; } + bool IsSupportedSpecification() const { return spec_.IsSupported(); } + + //! Static method to get the specification of any schema document + // Returns kDraftNone if document is silent + static const Specification GetSpecification(const ValueType& document) { + SchemaDraft draft = GetSchemaDraft(document); + if (draft != kDraftNone) + return Specification(draft); + else { + OpenApiVersion oapi = GetOpenApiVersion(document); + if (oapi != kVersionNone) + return Specification(oapi); + } + return Specification(kDraftNone); + } //! Get the root schema. const SchemaType& GetRoot() const { return *root_; } -private: + //! Gets the error object. + GValue& GetError() { return error_; } + const GValue& GetError() const { return error_; } + + static const StringRefType& GetSchemaErrorKeyword(SchemaErrorCode schemaErrorCode) { + switch (schemaErrorCode) { + case kSchemaErrorStartUnknown: return GetStartUnknownString(); + case kSchemaErrorRefPlainName: return GetRefPlainNameString(); + case kSchemaErrorRefInvalid: return GetRefInvalidString(); + case kSchemaErrorRefPointerInvalid: return GetRefPointerInvalidString(); + case kSchemaErrorRefUnknown: return GetRefUnknownString(); + case kSchemaErrorRefCyclical: return GetRefCyclicalString(); + case kSchemaErrorRefNoRemoteProvider: return GetRefNoRemoteProviderString(); + case kSchemaErrorRefNoRemoteSchema: return GetRefNoRemoteSchemaString(); + case kSchemaErrorRegexInvalid: return GetRegexInvalidString(); + case kSchemaErrorSpecUnknown: return GetSpecUnknownString(); + case kSchemaErrorSpecUnsupported: return GetSpecUnsupportedString(); + case kSchemaErrorSpecIllegal: return GetSpecIllegalString(); + case kSchemaErrorReadOnlyAndWriteOnly: return GetReadOnlyAndWriteOnlyString(); + default: return GetNullString(); + } + } + + //! Default error method + void SchemaError(const SchemaErrorCode code, const PointerType& location) { + currentError_ = GValue(kObjectType); + AddCurrentError(code, location); + } + + //! Method for error with single string value insert + void SchemaErrorValue(const SchemaErrorCode code, const PointerType& location, const Ch* value, SizeType length) { + currentError_ = GValue(kObjectType); + currentError_.AddMember(GetValueString(), GValue(value, length, *allocator_).Move(), *allocator_); + AddCurrentError(code, location); + } + + //! Method for error with invalid pointer + void SchemaErrorPointer(const SchemaErrorCode code, const PointerType& location, const Ch* value, SizeType length, const PointerType& pointer) { + currentError_ = GValue(kObjectType); + currentError_.AddMember(GetValueString(), GValue(value, length, *allocator_).Move(), *allocator_); + currentError_.AddMember(GetOffsetString(), static_cast(pointer.GetParseErrorOffset() / sizeof(Ch)), *allocator_); + AddCurrentError(code, location); + } + + private: //! Prohibit copying GenericSchemaDocument(const GenericSchemaDocument&); //! Prohibit assignment @@ -1738,6 +2019,146 @@ private: bool owned; }; + void AddErrorInstanceLocation(GValue& result, const PointerType& location) { + GenericStringBuffer sb; + location.StringifyUriFragment(sb); + GValue instanceRef(sb.GetString(), static_cast(sb.GetSize() / sizeof(Ch)), *allocator_); + result.AddMember(GetInstanceRefString(), instanceRef, *allocator_); + } + + void AddError(GValue& keyword, GValue& error) { + typename GValue::MemberIterator member = error_.FindMember(keyword); + if (member == error_.MemberEnd()) + error_.AddMember(keyword, error, *allocator_); + else { + if (member->value.IsObject()) { + GValue errors(kArrayType); + errors.PushBack(member->value, *allocator_); + member->value = errors; + } + member->value.PushBack(error, *allocator_); + } + } + + void AddCurrentError(const SchemaErrorCode code, const PointerType& location) { + RAPIDJSON_SCHEMA_PRINT(InvalidKeyword, GetSchemaErrorKeyword(code)); + currentError_.AddMember(GetErrorCodeString(), code, *allocator_); + AddErrorInstanceLocation(currentError_, location); + AddError(GValue(GetSchemaErrorKeyword(code)).Move(), currentError_); + } + +#define RAPIDJSON_STRING_(name, ...) \ + static const StringRefType& Get##name##String() {\ + static const Ch s[] = { __VA_ARGS__, '\0' };\ + static const StringRefType v(s, static_cast(sizeof(s) / sizeof(Ch) - 1)); \ + return v;\ + } + + RAPIDJSON_STRING_(InstanceRef, 'i', 'n', 's', 't', 'a', 'n', 'c', 'e', 'R', 'e', 'f') + RAPIDJSON_STRING_(ErrorCode, 'e', 'r', 'r', 'o', 'r', 'C', 'o', 'd', 'e') + RAPIDJSON_STRING_(Value, 'v', 'a', 'l', 'u', 'e') + RAPIDJSON_STRING_(Offset, 'o', 'f', 'f', 's', 'e', 't') + + RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') + RAPIDJSON_STRING_(SpecUnknown, 'S', 'p', 'e', 'c', 'U', 'n', 'k', 'n', 'o', 'w', 'n') + RAPIDJSON_STRING_(SpecUnsupported, 'S', 'p', 'e', 'c', 'U', 'n', 's', 'u', 'p', 'p', 'o', 'r', 't', 'e', 'd') + RAPIDJSON_STRING_(SpecIllegal, 'S', 'p', 'e', 'c', 'I', 'l', 'l', 'e', 'g', 'a', 'l') + RAPIDJSON_STRING_(StartUnknown, 'S', 't', 'a', 'r', 't', 'U', 'n', 'k', 'n', 'o', 'w', 'n') + RAPIDJSON_STRING_(RefPlainName, 'R', 'e', 'f', 'P', 'l', 'a', 'i', 'n', 'N', 'a', 'm', 'e') + RAPIDJSON_STRING_(RefInvalid, 'R', 'e', 'f', 'I', 'n', 'v', 'a', 'l', 'i', 'd') + RAPIDJSON_STRING_(RefPointerInvalid, 'R', 'e', 'f', 'P', 'o', 'i', 'n', 't', 'e', 'r', 'I', 'n', 'v', 'a', 'l', 'i', 'd') + RAPIDJSON_STRING_(RefUnknown, 'R', 'e', 'f', 'U', 'n', 'k', 'n', 'o', 'w', 'n') + RAPIDJSON_STRING_(RefCyclical, 'R', 'e', 'f', 'C', 'y', 'c', 'l', 'i', 'c', 'a', 'l') + RAPIDJSON_STRING_(RefNoRemoteProvider, 'R', 'e', 'f', 'N', 'o', 'R', 'e', 'm', 'o', 't', 'e', 'P', 'r', 'o', 'v', 'i', 'd', 'e', 'r') + RAPIDJSON_STRING_(RefNoRemoteSchema, 'R', 'e', 'f', 'N', 'o', 'R', 'e', 'm', 'o', 't', 'e', 'S', 'c', 'h', 'e', 'm', 'a') + RAPIDJSON_STRING_(ReadOnlyAndWriteOnly, 'R', 'e', 'a', 'd', 'O', 'n', 'l', 'y', 'A', 'n', 'd', 'W', 'r', 'i', 't', 'e', 'O', 'n', 'l', 'y') + RAPIDJSON_STRING_(RegexInvalid, 'R', 'e', 'g', 'e', 'x', 'I', 'n', 'v', 'a', 'l', 'i', 'd') + +#undef RAPIDJSON_STRING_ + + // Static method to get schema draft of any schema document + static SchemaDraft GetSchemaDraft(const ValueType& document) { + static const Ch kDraft03String[] = { 'h', 't', 't', 'p', ':', '/', '/', 'j', 's', 'o', 'n', '-', 's', 'c', 'h', 'e', 'm', 'a', '.', 'o', 'r', 'g', '/', 'd', 'r', 'a', 'f', 't', '-', '0', '3', '/', 's', 'c', 'h', 'e', 'm', 'a', '#', '\0' }; + static const Ch kDraft04String[] = { 'h', 't', 't', 'p', ':', '/', '/', 'j', 's', 'o', 'n', '-', 's', 'c', 'h', 'e', 'm', 'a', '.', 'o', 'r', 'g', '/', 'd', 'r', 'a', 'f', 't', '-', '0', '4', '/', 's', 'c', 'h', 'e', 'm', 'a', '#', '\0' }; + static const Ch kDraft05String[] = { 'h', 't', 't', 'p', ':', '/', '/', 'j', 's', 'o', 'n', '-', 's', 'c', 'h', 'e', 'm', 'a', '.', 'o', 'r', 'g', '/', 'd', 'r', 'a', 'f', 't', '-', '0', '5', '/', 's', 'c', 'h', 'e', 'm', 'a', '#', '\0' }; + static const Ch kDraft06String[] = { 'h', 't', 't', 'p', ':', '/', '/', 'j', 's', 'o', 'n', '-', 's', 'c', 'h', 'e', 'm', 'a', '.', 'o', 'r', 'g', '/', 'd', 'r', 'a', 'f', 't', '-', '0', '6', '/', 's', 'c', 'h', 'e', 'm', 'a', '#', '\0' }; + static const Ch kDraft07String[] = { 'h', 't', 't', 'p', ':', '/', '/', 'j', 's', 'o', 'n', '-', 's', 'c', 'h', 'e', 'm', 'a', '.', 'o', 'r', 'g', '/', 'd', 'r', 'a', 'f', 't', '-', '0', '7', '/', 's', 'c', 'h', 'e', 'm', 'a', '#', '\0' }; + static const Ch kDraft2019_09String[] = { 'h', 't', 't', 'p', 's', ':', '/', '/', 'j', 's', 'o', 'n', '-', 's', 'c', 'h', 'e', 'm', 'a', '.', 'o', 'r', 'g', '/', 'd', 'r', 'a', 'f', 't', '/', '2', '0', '1', '9', '-', '0', '9', '/', 's', 'c', 'h', 'e', 'm', 'a', '\0' }; + static const Ch kDraft2020_12String[] = { 'h', 't', 't', 'p', 's', ':', '/', '/', 'j', 's', 'o', 'n', '-', 's', 'c', 'h', 'e', 'm', 'a', '.', 'o', 'r', 'g', '/', 'd', 'r', 'a', 'f', 't', '/', '2', '0', '2', '0', '-', '1', '2', '/', 's', 'c', 'h', 'e', 'm', 'a', '\0' }; + + if (!document.IsObject()) { + return kDraftNone; + } + + // Get the schema draft from the $schema keyword at the supplied location + typename ValueType::ConstMemberIterator itr = document.FindMember(SchemaType::GetSchemaString()); + if (itr != document.MemberEnd()) { + if (!itr->value.IsString()) return kDraftUnknown; + const UriType draftUri(itr->value); + // Check base uri for match + if (draftUri.Match(UriType(kDraft04String), false)) return kDraft04; + if (draftUri.Match(UriType(kDraft05String), false)) return kDraft05; + if (draftUri.Match(UriType(kDraft06String), false)) return kDraft06; + if (draftUri.Match(UriType(kDraft07String), false)) return kDraft07; + if (draftUri.Match(UriType(kDraft03String), false)) return kDraft03; + if (draftUri.Match(UriType(kDraft2019_09String), false)) return kDraft2019_09; + if (draftUri.Match(UriType(kDraft2020_12String), false)) return kDraft2020_12; + return kDraftUnknown; + } + // $schema not found + return kDraftNone; + } + + + // Get open api version of any schema document + static OpenApiVersion GetOpenApiVersion(const ValueType& document) { + static const Ch kVersion20String[] = { '2', '.', '0', '\0' }; + static const Ch kVersion30String[] = { '3', '.', '0', '.', '\0' }; // ignore patch level + static const Ch kVersion31String[] = { '3', '.', '1', '.', '\0' }; // ignore patch level + static SizeType len = internal::StrLen(kVersion30String); + + if (!document.IsObject()) { + return kVersionNone; + } + + // Get the open api version from the swagger / openapi keyword at the supplied location + typename ValueType::ConstMemberIterator itr = document.FindMember(SchemaType::GetSwaggerString()); + if (itr == document.MemberEnd()) itr = document.FindMember(SchemaType::GetOpenApiString()); + if (itr != document.MemberEnd()) { + if (!itr->value.IsString()) return kVersionUnknown; + const ValueType kVersion20Value(kVersion20String); + if (kVersion20Value == itr->value) return kVersion20; // must match 2.0 exactly + const ValueType kVersion30Value(kVersion30String); + if (itr->value.GetStringLength() > len && kVersion30Value == ValueType(itr->value.GetString(), len)) return kVersion30; // must match 3.0.x + const ValueType kVersion31Value(kVersion31String); + if (itr->value.GetStringLength() > len && kVersion31Value == ValueType(itr->value.GetString(), len)) return kVersion31; // must match 3.1.x + return kVersionUnknown; + } + // swagger or openapi not found + return kVersionNone; + } + + // Get the draft of the schema or the open api version (which implies the draft). + // Report an error if schema draft or open api version not supported or not recognized, or both in document, and carry on. + void SetSchemaSpecification(const ValueType& document) { + // Look for '$schema', 'swagger' or 'openapi' keyword at document root + SchemaDraft docDraft = GetSchemaDraft(document); + OpenApiVersion docOapi = GetOpenApiVersion(document); + // Error if both in document + if (docDraft != kDraftNone && docOapi != kVersionNone) + SchemaError(kSchemaErrorSpecIllegal, PointerType()); + // Use document draft or open api version if present or use spec from constructor + if (docDraft != kDraftNone) + spec_ = Specification(docDraft); + else if (docOapi != kVersionNone) + spec_ = Specification(docOapi); + // Error if draft or version unknown + if (spec_.draft == kDraftUnknown || spec_.oapi == kVersionUnknown) + SchemaError(kSchemaErrorSpecUnknown, PointerType()); + else if (!spec_.IsSupported()) + SchemaError(kSchemaErrorSpecUnsupported, PointerType()); + } + // Changed by PR #1393 void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document, const UriType& id) { if (v.GetType() == kObjectType) { @@ -1754,6 +2175,9 @@ private: // Changed by PR #1393 const UriType& CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document, const UriType& id) { RAPIDJSON_ASSERT(pointer.IsValid()); + GenericStringBuffer sb; + pointer.StringifyUriFragment(sb); + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaDocument::CreateSchema", sb.GetString(), id.GetString()); if (v.IsObject()) { if (const SchemaType* sc = GetSchema(pointer)) { if (schema) @@ -1783,41 +2207,53 @@ private: if (itr == v.MemberEnd()) return false; + GenericStringBuffer sb; + source.StringifyUriFragment(sb); + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaDocument::HandleRefSchema", sb.GetString(), id.GetString()); // Resolve the source pointer to the $ref'ed schema (finally) new (schemaRef_.template Push()) SchemaRefPtr(&source); if (itr->value.IsString()) { SizeType len = itr->value.GetStringLength(); - if (len > 0) { + if (len == 0) + SchemaError(kSchemaErrorRefInvalid, source); + else { // First resolve $ref against the in-scope id UriType scopeId = UriType(id, allocator_); UriType ref = UriType(itr->value, allocator_).Resolve(scopeId, allocator_); + RAPIDJSON_SCHEMA_PRINT(SchemaIds, id.GetString(), itr->value.GetString(), ref.GetString()); // See if the resolved $ref minus the fragment matches a resolved id in this document // Search from the root. Returns the subschema in the document and its absolute JSON pointer. PointerType basePointer = PointerType(); const ValueType *base = FindId(document, ref, basePointer, docId_, false); if (!base) { // Remote reference - call the remote document provider - if (remoteProvider_) { - if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(ref)) { + if (!remoteProvider_) + SchemaError(kSchemaErrorRefNoRemoteProvider, source); + else { + if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(ref, spec_)) { const Ch* s = ref.GetFragString(); len = ref.GetFragStringLength(); if (len <= 1 || s[1] == '/') { // JSON pointer fragment, absolute in the remote schema const PointerType pointer(s, len, allocator_); - if (pointer.IsValid()) { + if (!pointer.IsValid()) + SchemaErrorPointer(kSchemaErrorRefPointerInvalid, source, s, len, pointer); + else { // Get the subschema if (const SchemaType *sc = remoteDocument->GetSchema(pointer)) { if (schema) *schema = sc; AddSchemaRefs(const_cast(sc)); return true; - } + } else + SchemaErrorValue(kSchemaErrorRefUnknown, source, ref.GetString(), ref.GetStringLength()); } - } else { - // Plain name fragment, not allowed - } - } + } else + // Plain name fragment, not allowed in remote schema + SchemaErrorValue(kSchemaErrorRefPlainName, source, s, len); + } else + SchemaErrorValue(kSchemaErrorRefNoRemoteSchema, source, ref.GetString(), ref.GetStringLength()); } } else { // Local reference @@ -1826,16 +2262,18 @@ private: if (len <= 1 || s[1] == '/') { // JSON pointer fragment, relative to the resolved URI const PointerType relPointer(s, len, allocator_); - if (relPointer.IsValid()) { + if (!relPointer.IsValid()) + SchemaErrorPointer(kSchemaErrorRefPointerInvalid, source, s, len, relPointer); + else { // Get the subschema if (const ValueType *pv = relPointer.Get(*base)) { // Now get the absolute JSON pointer by adding relative to base - PointerType pointer(basePointer); + PointerType pointer(basePointer, allocator_); for (SizeType i = 0; i < relPointer.GetTokenCount(); i++) pointer = pointer.Append(relPointer.GetTokens()[i], allocator_); - //GenericStringBuffer sb; - //pointer.StringifyUriFragment(sb); - if (pointer.IsValid() && !IsCyclicRef(pointer)) { + if (IsCyclicRef(pointer)) + SchemaErrorValue(kSchemaErrorRefCyclical, source, ref.GetString(), ref.GetStringLength()); + else { // Call CreateSchema recursively, but first compute the in-scope id for the $ref target as we have jumped there // TODO: cache pointer <-> id mapping size_t unresolvedTokenIndex; @@ -1843,17 +2281,21 @@ private: CreateSchema(schema, pointer, *pv, document, scopeId); return true; } - } + } else + SchemaErrorValue(kSchemaErrorRefUnknown, source, ref.GetString(), ref.GetStringLength()); } } else { // Plain name fragment, relative to the resolved URI + // Not supported in open api 2.0 and 3.0 + PointerType pointer(allocator_); + if (spec_.oapi == kVersion20 || spec_.oapi == kVersion30) + SchemaErrorValue(kSchemaErrorRefPlainName, source, s, len); // See if the fragment matches an id in this document. // Search from the base we just established. Returns the subschema in the document and its absolute JSON pointer. - PointerType pointer = PointerType(); - if (const ValueType *pv = FindId(*base, ref, pointer, UriType(ref.GetBaseString(), ref.GetBaseStringLength(), allocator_), true, basePointer)) { - if (!IsCyclicRef(pointer)) { - //GenericStringBuffer sb; - //pointer.StringifyUriFragment(sb); + else if (const ValueType *pv = FindId(*base, ref, pointer, UriType(ref.GetBaseString(), ref.GetBaseStringLength(), allocator_), true, basePointer)) { + if (IsCyclicRef(pointer)) + SchemaErrorValue(kSchemaErrorRefCyclical, source, ref.GetString(), ref.GetStringLength()); + else { // Call CreateSchema recursively, but first compute the in-scope id for the $ref target as we have jumped there // TODO: cache pointer <-> id mapping size_t unresolvedTokenIndex; @@ -1861,7 +2303,8 @@ private: CreateSchema(schema, pointer, *pv, document, scopeId); return true; } - } + } else + SchemaErrorValue(kSchemaErrorRefUnknown, source, ref.GetString(), ref.GetStringLength()); } } } @@ -1891,6 +2334,7 @@ private: } // See if it matches if (localuri.Match(finduri, full)) { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaDocument::FindId (match)", full ? localuri.GetString() : localuri.GetBaseString()); resval = const_cast(&doc); resptr = here; return resval; @@ -1917,6 +2361,7 @@ private: // Added by PR #1393 void AddSchemaRefs(SchemaType* schema) { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaDocument::AddSchemaRefs"); while (!schemaRef_.Empty()) { SchemaRefPtr *ref = schemaRef_.template Pop(1); SchemaEntry *entry = schemaMap_.template Push(); @@ -1958,8 +2403,11 @@ private: SchemaType* typeless_; internal::Stack schemaMap_; // Stores created Pointer -> Schemas internal::Stack schemaRef_; // Stores Pointer(s) from $ref(s) until resolved - SValue uri_; // Schema document URI + GValue uri_; // Schema document URI UriType docId_; + Specification spec_; + GValue error_; + GValue currentError_; }; //! GenericSchemaDocument using Value type. @@ -2023,11 +2471,10 @@ public: currentError_(), missingDependents_(), valid_(true), - flags_(kValidateDefaultFlags) -#if RAPIDJSON_SCHEMA_VERBOSE - , depth_(0) -#endif + flags_(kValidateDefaultFlags), + depth_(0) { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaValidator::GenericSchemaValidator"); } //! Constructor with output handler. @@ -2055,11 +2502,10 @@ public: currentError_(), missingDependents_(), valid_(true), - flags_(kValidateDefaultFlags) -#if RAPIDJSON_SCHEMA_VERBOSE - , depth_(0) -#endif + flags_(kValidateDefaultFlags), + depth_(0) { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaValidator::GenericSchemaValidator (output handler)"); } //! Destructor. @@ -2092,13 +2538,12 @@ public: return flags_; } - //! Checks whether the current state is valid. - // Implementation of ISchemaValidator virtual bool IsValid() const { if (!valid_) return false; if (GetContinueOnErrors() && !error_.ObjectEmpty()) return false; return true; } + //! End of Implementation of ISchemaValidator //! Gets the error object. ValueType& GetError() { return error_; } @@ -2114,7 +2559,7 @@ public: // If reporting all errors, the stack will be empty, so return "errors". const Ch* GetInvalidSchemaKeyword() const { if (!schemaStack_.Empty()) return CurrentContext().invalidKeyword; - if (GetContinueOnErrors() && !error_.ObjectEmpty()) return (const Ch*)GetErrorsString(); + if (GetContinueOnErrors() && !error_.ObjectEmpty()) return static_cast(GetErrorsString()); return 0; } @@ -2306,13 +2751,29 @@ public: void NoneOf(ISchemaValidator** subvalidators, SizeType count) { AddErrorArray(kValidateErrorAnyOf, subvalidators, count); } - void NotOneOf(ISchemaValidator** subvalidators, SizeType count, bool matched = false) { - AddErrorArray(matched ? kValidateErrorOneOfMatch : kValidateErrorOneOf, subvalidators, count); + void NotOneOf(ISchemaValidator** subvalidators, SizeType count) { + AddErrorArray(kValidateErrorOneOf, subvalidators, count); + } + void MultipleOneOf(SizeType index1, SizeType index2) { + ValueType matches(kArrayType); + matches.PushBack(index1, GetStateAllocator()); + matches.PushBack(index2, GetStateAllocator()); + currentError_.SetObject(); + currentError_.AddMember(GetMatchesString(), matches, GetStateAllocator()); + AddCurrentError(kValidateErrorOneOfMatch); } void Disallowed() { currentError_.SetObject(); AddCurrentError(kValidateErrorNot); } + void DisallowedWhenWriting() { + currentError_.SetObject(); + AddCurrentError(kValidateErrorReadOnly); + } + void DisallowedWhenReading() { + currentError_.SetObject(); + AddCurrentError(kValidateErrorWriteOnly); + } #define RAPIDJSON_STRING_(name, ...) \ static const StringRefType& Get##name##String() {\ @@ -2331,25 +2792,18 @@ public: RAPIDJSON_STRING_(ErrorCode, 'e', 'r', 'r', 'o', 'r', 'C', 'o', 'd', 'e') RAPIDJSON_STRING_(ErrorMessage, 'e', 'r', 'r', 'o', 'r', 'M', 'e', 's', 's', 'a', 'g', 'e') RAPIDJSON_STRING_(Duplicates, 'd', 'u', 'p', 'l', 'i', 'c', 'a', 't', 'e', 's') + RAPIDJSON_STRING_(Matches, 'm', 'a', 't', 'c', 'h', 'e', 's') #undef RAPIDJSON_STRING_ -#if RAPIDJSON_SCHEMA_VERBOSE -#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ -RAPIDJSON_MULTILINEMACRO_BEGIN\ - *documentStack_.template Push() = '\0';\ - documentStack_.template Pop(1);\ - internal::PrintInvalidDocument(documentStack_.template Bottom());\ -RAPIDJSON_MULTILINEMACRO_END -#else -#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() -#endif - #define RAPIDJSON_SCHEMA_HANDLE_BEGIN_(method, arg1)\ if (!valid_) return false; \ if ((!BeginValue() && !GetContinueOnErrors()) || (!CurrentSchema().method arg1 && !GetContinueOnErrors())) {\ - RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_();\ - return valid_ = false;\ + *documentStack_.template Push() = '\0';\ + documentStack_.template Pop(1);\ + RAPIDJSON_SCHEMA_PRINT(InvalidDocument, documentStack_.template Bottom());\ + valid_ = false;\ + return valid_;\ } #define RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2)\ @@ -2386,52 +2840,68 @@ RAPIDJSON_MULTILINEMACRO_END { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } bool StartObject() { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaValidator::StartObject"); RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); - return valid_ = !outputHandler_ || outputHandler_->StartObject(); + valid_ = !outputHandler_ || outputHandler_->StartObject(); + return valid_; } bool Key(const Ch* str, SizeType len, bool copy) { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaValidator::Key", str); if (!valid_) return false; AppendToken(str, len); - if (!CurrentSchema().Key(CurrentContext(), str, len, copy) && !GetContinueOnErrors()) return valid_ = false; + if (!CurrentSchema().Key(CurrentContext(), str, len, copy) && !GetContinueOnErrors()) { + valid_ = false; + return valid_; + } RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); - return valid_ = !outputHandler_ || outputHandler_->Key(str, len, copy); + valid_ = !outputHandler_ || outputHandler_->Key(str, len, copy); + return valid_; } bool EndObject(SizeType memberCount) { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaValidator::EndObject"); if (!valid_) return false; RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); - if (!CurrentSchema().EndObject(CurrentContext(), memberCount) && !GetContinueOnErrors()) return valid_ = false; + if (!CurrentSchema().EndObject(CurrentContext(), memberCount) && !GetContinueOnErrors()) { + valid_ = false; + return valid_; + } RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); } bool StartArray() { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaValidator::StartArray"); RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); - return valid_ = !outputHandler_ || outputHandler_->StartArray(); + valid_ = !outputHandler_ || outputHandler_->StartArray(); + return valid_; } bool EndArray(SizeType elementCount) { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaValidator::EndArray"); if (!valid_) return false; RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); - if (!CurrentSchema().EndArray(CurrentContext(), elementCount) && !GetContinueOnErrors()) return valid_ = false; + if (!CurrentSchema().EndArray(CurrentContext(), elementCount) && !GetContinueOnErrors()) { + valid_ = false; + return valid_; + } RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); } -#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_ #undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_ #undef RAPIDJSON_SCHEMA_HANDLE_PARALLEL_ #undef RAPIDJSON_SCHEMA_HANDLE_VALUE_ // Implementation of ISchemaStateFactory virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root, const bool inheritContinueOnErrors) { + *documentStack_.template Push() = '\0'; + documentStack_.template Pop(1); ISchemaValidator* sv = new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, documentStack_.template Bottom(), documentStack_.GetSize(), -#if RAPIDJSON_SCHEMA_VERBOSE depth_ + 1, -#endif &GetStateAllocator()); - sv->SetValidateFlags(inheritContinueOnErrors ? GetValidateFlags() : GetValidateFlags() & ~(unsigned)kValidateContinueOnErrorFlag); + sv->SetValidateFlags(inheritContinueOnErrors ? GetValidateFlags() : GetValidateFlags() & ~static_cast(kValidateContinueOnErrorFlag)); return sv; } @@ -2462,6 +2932,7 @@ RAPIDJSON_MULTILINEMACRO_END virtual void FreeState(void* p) { StateAllocator::Free(p); } + // End of implementation of ISchemaStateFactory private: typedef typename SchemaType::Context Context; @@ -2472,9 +2943,7 @@ private: const SchemaDocumentType& schemaDocument, const SchemaType& root, const char* basePath, size_t basePathSize, -#if RAPIDJSON_SCHEMA_VERBOSE unsigned depth, -#endif StateAllocator* allocator = 0, size_t schemaStackCapacity = kDefaultSchemaStackCapacity, size_t documentStackCapacity = kDefaultDocumentStackCapacity) @@ -2490,11 +2959,10 @@ private: currentError_(), missingDependents_(), valid_(true), - flags_(kValidateDefaultFlags) -#if RAPIDJSON_SCHEMA_VERBOSE - , depth_(depth) -#endif + flags_(kValidateDefaultFlags), + depth_(depth) { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaValidator::GenericSchemaValidator (internal)", basePath && basePathSize ? basePath : ""); if (basePath && basePathSize) memcpy(documentStack_.template Push(basePathSize), basePath, basePathSize); } @@ -2510,6 +2978,7 @@ private: } bool BeginValue() { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaValidator::BeginValue"); if (schemaStack_.Empty()) PushSchema(root_); else { @@ -2531,6 +3000,7 @@ private: ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; va = static_cast(MallocState(sizeof(ISchemaValidator*) * count)); + std::memset(va, 0, sizeof(ISchemaValidator*) * count); for (SizeType i = 0; i < count; i++) va[validatorCount++] = CreateSchemaValidator(*sa[i], true); // Inherit continueOnError } @@ -2541,17 +3011,15 @@ private: } bool EndValue() { + RAPIDJSON_SCHEMA_PRINT(Method, "GenericSchemaValidator::EndValue"); if (!CurrentSchema().EndValue(CurrentContext()) && !GetContinueOnErrors()) return false; -#if RAPIDJSON_SCHEMA_VERBOSE GenericStringBuffer sb; - schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); - + schemaDocument_->GetPointer(&CurrentSchema()).StringifyUriFragment(sb); *documentStack_.template Push() = '\0'; documentStack_.template Pop(1); - internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom()); -#endif + RAPIDJSON_SCHEMA_PRINT(ValidatorPointers, sb.GetString(), documentStack_.template Bottom(), depth_); void* hasher = CurrentContext().hasher; uint64_t h = hasher && CurrentContext().arrayUniqueness ? static_cast(hasher)->GetHashCode() : 0; @@ -2602,7 +3070,7 @@ private: } } - RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, *this, &schema); } + RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, *this, &schema, flags_); } RAPIDJSON_FORCEINLINE void PopSchema() { Context* c = schemaStack_.template Pop(1); @@ -2704,9 +3172,7 @@ private: ValueType missingDependents_; bool valid_; unsigned flags_; -#if RAPIDJSON_SCHEMA_VERBOSE unsigned depth_; -#endif }; typedef GenericSchemaValidator SchemaValidator; diff --git a/src/3rdparty/rapidjson/uri.h b/src/3rdparty/rapidjson/uri.h index 7de7b805..f93e508a 100644 --- a/src/3rdparty/rapidjson/uri.h +++ b/src/3rdparty/rapidjson/uri.h @@ -238,20 +238,27 @@ private: // Allocate one block containing each part of the URI (5) plus base plus full URI, all null terminated. // Order: scheme, auth, path, query, frag, base, uri + // Note need to set, increment, assign in 3 stages to avoid compiler warning bug. size_t total = (3 * len + 7) * sizeof(Ch); scheme_ = static_cast(allocator_->Malloc(total)); *scheme_ = '\0'; - auth_ = scheme_ + 1; + auth_ = scheme_; + auth_++; *auth_ = '\0'; - path_ = auth_ + 1; + path_ = auth_; + path_++; *path_ = '\0'; - query_ = path_ + 1; + query_ = path_; + query_++; *query_ = '\0'; - frag_ = query_ + 1; + frag_ = query_; + frag_++; *frag_ = '\0'; - base_ = frag_ + 1; + base_ = frag_; + base_++; *base_ = '\0'; - uri_ = base_ + 1; + uri_ = base_; + uri_++; *uri_ = '\0'; return total; } @@ -293,7 +300,9 @@ private: } } // Look for auth (//([^/?#]*))? - auth_ = scheme_ + GetSchemeStringLength() + 1; + // Note need to set, increment, assign in 3 stages to avoid compiler warning bug. + auth_ = scheme_ + GetSchemeStringLength(); + auth_++; *auth_ = '\0'; if (start < len - 1 && uri[start] == '/' && uri[start + 1] == '/') { pos2 = start + 2; @@ -308,7 +317,9 @@ private: start = pos2; } // Look for path ([^?#]*) - path_ = auth_ + GetAuthStringLength() + 1; + // Note need to set, increment, assign in 3 stages to avoid compiler warning bug. + path_ = auth_ + GetAuthStringLength(); + path_++; *path_ = '\0'; if (start < len) { pos2 = start; @@ -326,7 +337,9 @@ private: } } // Look for query (\?([^#]*))? - query_ = path_ + GetPathStringLength() + 1; + // Note need to set, increment, assign in 3 stages to avoid compiler warning bug. + query_ = path_ + GetPathStringLength(); + query_++; *query_ = '\0'; if (start < len && uri[start] == '?') { pos2 = start + 1; @@ -341,7 +354,9 @@ private: } } // Look for fragment (#(.*))? - frag_ = query_ + GetQueryStringLength() + 1; + // Note need to set, increment, assign in 3 stages to avoid compiler warning bug. + frag_ = query_ + GetQueryStringLength(); + frag_++; *frag_ = '\0'; if (start < len && uri[start] == '#') { std::memcpy(frag_, &uri[start], (len - start) * sizeof(Ch)); diff --git a/src/3rdparty/rapidjson/writer.h b/src/3rdparty/rapidjson/writer.h index f47005a1..6427c3e8 100644 --- a/src/3rdparty/rapidjson/writer.h +++ b/src/3rdparty/rapidjson/writer.h @@ -67,6 +67,7 @@ enum WriteFlag { kWriteNoFlags = 0, //!< No flags are set. kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. kWriteNanAndInfFlag = 2, //!< Allow writing of Infinity, -Infinity and NaN. + kWriteNanAndInfNullFlag = 4, //!< Allow writing of Infinity, -Infinity and NaN as null. kWriteDefaultFlags = RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining RAPIDJSON_WRITE_DEFAULT_FLAGS }; @@ -349,8 +350,13 @@ protected: bool WriteDouble(double d) { if (internal::Double(d).IsNanOrInf()) { - if (!(writeFlags & kWriteNanAndInfFlag)) + if (!(writeFlags & kWriteNanAndInfFlag) && !(writeFlags & kWriteNanAndInfNullFlag)) return false; + if (writeFlags & kWriteNanAndInfNullFlag) { + PutReserve(*os_, 4); + PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); + return true; + } if (internal::Double(d).IsNan()) { PutReserve(*os_, 3); PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); @@ -549,6 +555,11 @@ inline bool Writer::WriteDouble(double d) { // Note: This code path can only be reached if (RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) return false; + if (kWriteDefaultFlags & kWriteNanAndInfNullFlag) { + PutReserve(*os_, 4); + PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); + return true; + } if (internal::Double(d).IsNan()) { PutReserve(*os_, 3); PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); diff --git a/src/backend/common/Hashrate.cpp b/src/backend/common/Hashrate.cpp index 2d8c8ce2..a5102fd1 100644 --- a/src/backend/common/Hashrate.cpp +++ b/src/backend/common/Hashrate.cpp @@ -30,10 +30,10 @@ #include "base/tools/Handle.h" -inline static const char *format(double h, char *buf, size_t size) +inline static const char *format(std::pair h, char *buf, size_t size) { - if (std::isnormal(h)) { - snprintf(buf, size, (h < 100.0) ? "%04.2f" : "%03.1f", h); + if (h.first) { + snprintf(buf, size, (h.second < 100.0) ? "%04.2f" : "%03.1f", h.second); return buf; } @@ -80,15 +80,16 @@ double xmrig::Hashrate::average() const } -const char *xmrig::Hashrate::format(double h, char *buf, size_t size) +const char *xmrig::Hashrate::format(std::pair h, char *buf, size_t size) { return ::format(h, buf, size); } -rapidjson::Value xmrig::Hashrate::normalize(double d) +rapidjson::Value xmrig::Hashrate::normalize(std::pair d) { - return Json::normalize(d, false); + using namespace rapidjson; + return d.first ? Value(floor(d.second * 100.0) / 100.0) : Value(kNullType); } @@ -122,11 +123,11 @@ rapidjson::Value xmrig::Hashrate::toJSON(size_t threadId, rapidjson::Document &d #endif -double xmrig::Hashrate::hashrate(size_t index, size_t ms) const +std::pair xmrig::Hashrate::hashrate(size_t index, size_t ms) const { assert(index < m_threads); if (index >= m_threads) { - return nan(""); + return { false, 0.0 }; } uint64_t earliestHashCount = 0; @@ -157,17 +158,27 @@ double xmrig::Hashrate::hashrate(size_t index, size_t ms) const } while (idx != idx_start); if (!haveFullSet || earliestStamp == 0 || lastestStamp == 0) { - return nan(""); + return { false, 0.0 }; } - if (lastestStamp - earliestStamp == 0) { - return nan(""); + if (lastestHashCnt == earliestHashCount) { + return { true, 0.0 }; + } + + if (lastestStamp == earliestStamp) { + return { false, 0.0 }; } const auto hashes = static_cast(lastestHashCnt - earliestHashCount); - const auto time = static_cast(lastestStamp - earliestStamp) / 1000.0; + const auto time = static_cast(lastestStamp - earliestStamp); - return hashes / time; + const auto hr = hashes * 1000.0 / time; + + if (!std::isnormal(hr)) { + return { false, 0.0 }; + } + + return { true, hr }; } diff --git a/src/backend/common/Hashrate.h b/src/backend/common/Hashrate.h index 7d753a3d..10859741 100644 --- a/src/backend/common/Hashrate.h +++ b/src/backend/common/Hashrate.h @@ -47,16 +47,16 @@ public: Hashrate(size_t threads); ~Hashrate(); - inline double calc(size_t ms) const { const double data = hashrate(0U, ms); return std::isnormal(data) ? data : 0.0; } - inline double calc(size_t threadId, size_t ms) const { return hashrate(threadId + 1, ms); } + inline std::pair calc(size_t ms) const { return hashrate(0U, ms); } + inline std::pair calc(size_t threadId, size_t ms) const { return hashrate(threadId + 1, ms); } inline size_t threads() const { return m_threads > 0U ? m_threads - 1U : 0U; } inline void add(size_t threadId, uint64_t count, uint64_t timestamp) { addData(threadId + 1U, count, timestamp); } inline void add(uint64_t count, uint64_t timestamp) { addData(0U, count, timestamp); } double average() const; - static const char *format(double h, char *buf, size_t size); - static rapidjson::Value normalize(double d); + static const char *format(std::pair h, char *buf, size_t size); + static rapidjson::Value normalize(std::pair d); # ifdef XMRIG_FEATURE_API rapidjson::Value toJSON(rapidjson::Document &doc) const; @@ -64,7 +64,7 @@ public: # endif private: - double hashrate(size_t index, size_t ms) const; + std::pair hashrate(size_t index, size_t ms) const; void addData(size_t index, uint64_t count, uint64_t timestamp); constexpr static size_t kBucketSize = 2 << 11; diff --git a/src/backend/common/misc/PciTopology.h b/src/backend/common/misc/PciTopology.h index af1844c9..acef4d91 100644 --- a/src/backend/common/misc/PciTopology.h +++ b/src/backend/common/misc/PciTopology.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2024 SChernykh + * Copyright (c) 2016-2024 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,10 +19,8 @@ #ifndef XMRIG_PCITOPOLOGY_H #define XMRIG_PCITOPOLOGY_H - #include - #include "base/tools/String.h" @@ -33,7 +31,14 @@ class PciTopology { public: PciTopology() = default; - PciTopology(uint32_t bus, uint32_t device, uint32_t function) : m_valid(true), m_bus(bus), m_device(device), m_function(function) {} + + template + inline PciTopology(T bus, T device, T function) + : m_valid(true), + m_bus(static_cast(bus)), + m_device(static_cast(device)), + m_function(static_cast(function)) + {} inline bool isEqual(const PciTopology &other) const { return m_valid == other.m_valid && toUint32() == other.toUint32(); } inline bool isValid() const { return m_valid; } @@ -70,4 +75,4 @@ private: } // namespace xmrig -#endif /* XMRIG_PCITOPOLOGY_H */ +#endif // XMRIG_PCITOPOLOGY_H diff --git a/src/backend/cpu/CpuWorker.cpp b/src/backend/cpu/CpuWorker.cpp index d0047e4b..bef2e898 100644 --- a/src/backend/cpu/CpuWorker.cpp +++ b/src/backend/cpu/CpuWorker.cpp @@ -359,7 +359,9 @@ void xmrig::CpuWorker::start() } } - consumeJob(); + if (!Nonce::isPaused()) { + consumeJob(); + } } } diff --git a/src/backend/cpu/cpu.cmake b/src/backend/cpu/cpu.cmake index c485683a..f9a02abd 100644 --- a/src/backend/cpu/cpu.cmake +++ b/src/backend/cpu/cpu.cmake @@ -49,8 +49,15 @@ endif() if (XMRIG_ARM) list(APPEND SOURCES_BACKEND_CPU src/backend/cpu/platform/BasicCpuInfo_arm.cpp) - if (XMRIG_OS_UNIX) - list(APPEND SOURCES_BACKEND_CPU src/backend/cpu/platform/lscpu_arm.cpp) + if (XMRIG_OS_WIN) + list(APPEND SOURCES_BACKEND_CPU src/backend/cpu/platform/BasicCpuInfo_arm_win.cpp) + elseif(XMRIG_OS_APPLE) + list(APPEND SOURCES_BACKEND_CPU src/backend/cpu/platform/BasicCpuInfo_arm_mac.cpp) + else() + list(APPEND SOURCES_BACKEND_CPU + src/backend/cpu/platform/lscpu_arm.cpp + src/backend/cpu/platform/BasicCpuInfo_arm_unix.cpp + ) endif() else() list(APPEND SOURCES_BACKEND_CPU src/backend/cpu/platform/BasicCpuInfo.cpp) diff --git a/src/backend/cpu/platform/BasicCpuInfo.h b/src/backend/cpu/platform/BasicCpuInfo.h index f56d5425..5ea5661d 100644 --- a/src/backend/cpu/platform/BasicCpuInfo.h +++ b/src/backend/cpu/platform/BasicCpuInfo.h @@ -1,7 +1,7 @@ /* XMRig * Copyright (c) 2017-2019 XMR-Stak , - * Copyright (c) 2018-2023 SChernykh - * Copyright (c) 2016-2023 XMRig + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -85,6 +85,8 @@ private: uint32_t m_family = 0; uint32_t m_model = 0; uint32_t m_stepping = 0; +# else + void init_arm(); # endif Assembly m_assembly = Assembly::NONE; diff --git a/src/backend/cpu/platform/BasicCpuInfo_arm.cpp b/src/backend/cpu/platform/BasicCpuInfo_arm.cpp index b684c510..a38c5172 100644 --- a/src/backend/cpu/platform/BasicCpuInfo_arm.cpp +++ b/src/backend/cpu/platform/BasicCpuInfo_arm.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,44 +16,15 @@ * along with this program. If not, see . */ -#include "base/tools/String.h" - - #include #include -#include #include -#if __ARM_FEATURE_CRYPTO && !defined(__APPLE__) -# include -# if !defined(XMRIG_OS_FREEBSD) -# include -# else -# include -# include -# ifndef ID_AA64ISAR0_AES_VAL -# define ID_AA64ISAR0_AES_VAL ID_AA64ISAR0_AES -# endif -# endif -#endif - - #include "backend/cpu/platform/BasicCpuInfo.h" #include "3rdparty/rapidjson/document.h" -#if defined(XMRIG_OS_UNIX) -namespace xmrig { - -extern String cpu_name_arm(); - -} // namespace xmrig -#elif defined(XMRIG_OS_MACOS) -# include -#endif - - xmrig::BasicCpuInfo::BasicCpuInfo() : m_threads(std::thread::hardware_concurrency()) { @@ -68,28 +39,7 @@ xmrig::BasicCpuInfo::BasicCpuInfo() : memcpy(m_brand, "ARMv7", 5); # endif -# if __ARM_FEATURE_CRYPTO -# if defined(__APPLE__) - m_flags.set(FLAG_AES, true); -# elif defined(XMRIG_OS_FREEBSD) - uint64_t isar0 = READ_SPECIALREG(id_aa64isar0_el1); - m_flags.set(FLAG_AES, ID_AA64ISAR0_AES_VAL(isar0) >= ID_AA64ISAR0_AES_BASE); -# else - m_flags.set(FLAG_AES, getauxval(AT_HWCAP) & HWCAP_AES); -# endif -# endif - -# if defined(XMRIG_OS_UNIX) - auto name = cpu_name_arm(); - if (!name.isNull()) { - strncpy(m_brand, name, sizeof(m_brand) - 1); - } - - m_flags.set(FLAG_PDPE1GB, std::ifstream("/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages").good()); -# elif defined(XMRIG_OS_MACOS) - size_t buflen = sizeof(m_brand); - sysctlbyname("machdep.cpu.brand_string", &m_brand, &buflen, nullptr, 0); -# endif + init_arm(); } diff --git a/src/backend/cpu/platform/BasicCpuInfo_arm_mac.cpp b/src/backend/cpu/platform/BasicCpuInfo_arm_mac.cpp new file mode 100644 index 00000000..f1c35c6f --- /dev/null +++ b/src/backend/cpu/platform/BasicCpuInfo_arm_mac.cpp @@ -0,0 +1,32 @@ +/* XMRig + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "backend/cpu/platform/BasicCpuInfo.h" + +#include + + +void xmrig::BasicCpuInfo::init_arm() +{ +# if __ARM_FEATURE_CRYPTO + m_flags.set(FLAG_AES, true); // FIXME +# endif + + size_t buflen = sizeof(m_brand); + sysctlbyname("machdep.cpu.brand_string", &m_brand, &buflen, nullptr, 0); +} diff --git a/src/backend/cpu/platform/BasicCpuInfo_arm_unix.cpp b/src/backend/cpu/platform/BasicCpuInfo_arm_unix.cpp new file mode 100644 index 00000000..d480e7c5 --- /dev/null +++ b/src/backend/cpu/platform/BasicCpuInfo_arm_unix.cpp @@ -0,0 +1,68 @@ +/* XMRig + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "backend/cpu/platform/BasicCpuInfo.h" +#include "base/tools/String.h" + + +#include + + +#if __ARM_FEATURE_CRYPTO +# include +# if !defined(XMRIG_OS_FREEBSD) +# include +# else +# include +# include +# ifndef ID_AA64ISAR0_AES_VAL +# define ID_AA64ISAR0_AES_VAL ID_AA64ISAR0_AES +# endif +# endif +#endif + + +namespace xmrig { + + +extern String cpu_name_arm(); + + +} // namespace xmrig + + +void xmrig::BasicCpuInfo::init_arm() +{ +# if __ARM_FEATURE_CRYPTO +# if defined(XMRIG_OS_FREEBSD) + uint64_t isar0 = READ_SPECIALREG(id_aa64isar0_el1); + m_flags.set(FLAG_AES, ID_AA64ISAR0_AES_VAL(isar0) >= ID_AA64ISAR0_AES_BASE); +# else + m_flags.set(FLAG_AES, getauxval(AT_HWCAP) & HWCAP_AES); +# endif +# endif + +# if defined(XMRIG_OS_UNIX) + auto name = cpu_name_arm(); + if (!name.isNull()) { + strncpy(m_brand, name, sizeof(m_brand) - 1); + } + + m_flags.set(FLAG_PDPE1GB, std::ifstream("/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages").good()); +# endif +} diff --git a/src/backend/cpu/platform/BasicCpuInfo_arm_win.cpp b/src/backend/cpu/platform/BasicCpuInfo_arm_win.cpp new file mode 100644 index 00000000..a11aa94d --- /dev/null +++ b/src/backend/cpu/platform/BasicCpuInfo_arm_win.cpp @@ -0,0 +1,32 @@ +/* XMRig + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "backend/cpu/platform/BasicCpuInfo.h" + +#include + + +void xmrig::BasicCpuInfo::init_arm() +{ + DWORD size = sizeof(m_brand) - 1; + const char *subkey = "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0"; + + RegGetValueA(HKEY_LOCAL_MACHINE, subkey, "ProcessorNameString", RRF_RT_REG_SZ, nullptr, m_brand, &size); + + m_flags.set(FLAG_AES, IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE)); +} diff --git a/src/backend/cpu/platform/HwlocCpuInfo.cpp b/src/backend/cpu/platform/HwlocCpuInfo.cpp index f6137c31..f796416b 100644 --- a/src/backend/cpu/platform/HwlocCpuInfo.cpp +++ b/src/backend/cpu/platform/HwlocCpuInfo.cpp @@ -311,22 +311,41 @@ void xmrig::HwlocCpuInfo::processTopLevelCache(hwloc_obj_t cache, const Algorith uint32_t intensity = algorithm.maxIntensity() == 1 ? 0 : 1; if (cache->attr->cache.depth == 3) { - for (size_t i = 0; i < cache->arity; ++i) { - hwloc_obj_t l2 = cache->children[i]; + auto process_L2 = [&L2, &L2_associativity, L3_exclusive, this, &extra, scratchpad](hwloc_obj_t l2) { if (!hwloc_obj_type_is_cache(l2->type) || l2->attr == nullptr) { - continue; + return; } L2 += l2->attr->cache.size; L2_associativity = l2->attr->cache.associativity; - if (L3_exclusive && l2->attr->cache.size >= scratchpad) { - extra += scratchpad; + if (L3_exclusive) { + if ((vendor() == VENDOR_AMD) && ((arch() == ARCH_ZEN4) || (arch() == ARCH_ZEN5))) { + // Use extra L2 only on newer CPUs because older CPUs (Zen 3 and older) don't benefit from it. + // For some reason, AMD CPUs can use only half of the exclusive L2/L3 cache combo efficiently + extra += std::min(l2->attr->cache.size / 2, scratchpad); + } + else if (l2->attr->cache.size >= scratchpad) { + extra += scratchpad; + } + } + }; + + for (size_t i = 0; i < cache->arity; ++i) { + hwloc_obj_t ch = cache->children[i]; + if (ch->type == HWLOC_OBJ_GROUP) { + for (size_t j = 0; j < ch->arity; ++j) { + process_L2(ch->children[j]); + } + } + else { + process_L2(ch); } } } - if (scratchpad == 2 * oneMiB) { + // This code is supposed to run only on Intel CPUs + if ((vendor() == VENDOR_INTEL) && (scratchpad == 2 * oneMiB)) { if (L2 && (cores.size() * oneMiB) == L2 && L2_associativity == 16 && L3 >= L2) { L3 = L2; extra = L2; @@ -341,7 +360,7 @@ void xmrig::HwlocCpuInfo::processTopLevelCache(hwloc_obj_t cache, const Algorith } # ifdef XMRIG_ALGO_RANDOMX - if ((algorithm.family() == Algorithm::RANDOM_X) && L3_exclusive && (PUs > cores.size()) && (PUs < cores.size() * 2)) { + if ((vendor() == VENDOR_INTEL) && (algorithm.family() == Algorithm::RANDOM_X) && L3_exclusive && (PUs < cores.size() * 2)) { // Use all L3+L2 on latest Intel CPUs with P-cores, E-cores and exclusive L3 cache cacheHashes = (L3 + L2) / scratchpad; } diff --git a/src/backend/cpu/platform/lscpu_arm.cpp b/src/backend/cpu/platform/lscpu_arm.cpp index 7616632b..31658e17 100644 --- a/src/backend/cpu/platform/lscpu_arm.cpp +++ b/src/backend/cpu/platform/lscpu_arm.cpp @@ -1,7 +1,7 @@ /* XMRig * Copyright (c) 2018 Riku Voipio - * Copyright (c) 2018-2023 SChernykh - * Copyright (c) 2016-2023 XMRig + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -98,8 +98,11 @@ static const id_part arm_part[] = { { 0xd0e, "Cortex-A76AE" }, { 0xd13, "Cortex-R52" }, { 0xd15, "Cortex-R82" }, + { 0xd16, "Cortex-R52+" }, { 0xd20, "Cortex-M23" }, { 0xd21, "Cortex-M33" }, + { 0xd22, "Cortex-M55" }, + { 0xd23, "Cortex-M85" }, { 0xd40, "Neoverse-V1" }, { 0xd41, "Cortex-A78" }, { 0xd42, "Cortex-A78AE" }, @@ -115,6 +118,17 @@ static const id_part arm_part[] = { { 0xd4d, "Cortex-A715" }, { 0xd4e, "Cortex-X3" }, { 0xd4f, "Neoverse-V2" }, + { 0xd80, "Cortex-A520" }, + { 0xd81, "Cortex-A720" }, + { 0xd82, "Cortex-X4" }, + { 0xd83, "Neoverse-V3AE" }, + { 0xd84, "Neoverse-V3" }, + { 0xd85, "Cortex-X925" }, + { 0xd87, "Cortex-A725" }, + { 0xd88, "Cortex-A520AE" }, + { 0xd89, "Cortex-A720AE" }, + { 0xd8e, "Neoverse-N3" }, + { 0xd8f, "Cortex-A320" }, { -1, nullptr } }; @@ -154,6 +168,7 @@ static const id_part apm_part[] = { }; static const id_part qcom_part[] = { + { 0x001, "Oryon" }, { 0x00f, "Scorpion" }, { 0x02d, "Scorpion" }, { 0x04d, "Krait" }, @@ -194,6 +209,22 @@ static const id_part marvell_part[] = { { -1, nullptr } }; +static const id_part apple_part[] = { + { 0x022, "M1" }, + { 0x023, "M1" }, + { 0x024, "M1-Pro" }, + { 0x025, "M1-Pro" }, + { 0x028, "M1-Max" }, + { 0x029, "M1-Max" }, + { 0x032, "M2" }, + { 0x033, "M2" }, + { 0x034, "M2-Pro" }, + { 0x035, "M2-Pro" }, + { 0x038, "M2-Max" }, + { 0x039, "M2-Max" }, + { -1, nullptr } +}; + static const id_part faraday_part[] = { { 0x526, "FA526" }, { 0x626, "FA626" }, @@ -227,47 +258,40 @@ static const id_part intel_part[] = { static const struct id_part fujitsu_part[] = { { 0x001, "A64FX" }, + { 0x003, "MONAKA" }, { -1, nullptr } }; static const id_part hisi_part[] = { - { 0xd01, "Kunpeng-920" }, /* aka tsv110 */ - { 0xd40, "Cortex-A76" }, /* HiSilicon uses this ID though advertises A76 */ + { 0xd01, "TaiShan-v110" }, /* used in Kunpeng-920 SoC */ + { 0xd02, "TaiShan-v120" }, /* used in Kirin 990A and 9000S SoCs */ + { 0xd40, "Cortex-A76" }, /* HiSilicon uses this ID though advertises A76 */ + { 0xd41, "Cortex-A77" }, /* HiSilicon uses this ID though advertises A77 */ { -1, nullptr } }; -static const id_part apple_part[] = { - { 0x022, "M1" }, - { 0x023, "M1" }, - { 0x024, "M1-Pro" }, - { 0x025, "M1-Pro" }, - { 0x028, "M1-Max" }, - { 0x029, "M1-Max" }, - { 0x032, "M2" }, - { 0x033, "M2" }, - { 0x034, "M2-Pro" }, - { 0x035, "M2-Pro" }, - { 0x038, "M2-Max" }, - { 0x039, "M2-Max" }, - { -1, nullptr } -}; - - -static const struct id_part ft_part[] = { - { 0x660, "FTC660" }, - { 0x661, "FTC661" }, - { 0x662, "FTC662" }, - { 0x663, "FTC663" }, - { -1, nullptr } -}; - - static const struct id_part ampere_part[] = { { 0xac3, "Ampere-1" }, { 0xac4, "Ampere-1a" }, { -1, nullptr } }; +static const struct id_part ft_part[] = { + { 0x303, "FTC310" }, + { 0x660, "FTC660" }, + { 0x661, "FTC661" }, + { 0x662, "FTC662" }, + { 0x663, "FTC663" }, + { 0x664, "FTC664" }, + { 0x862, "FTC862" }, + { -1, nullptr } +}; + +static const struct id_part ms_part[] = { + { 0xd49, "Azure-Cobalt-100" }, + { -1, nullptr } +}; + static const hw_impl hw_implementer[] = { { 0x41, arm_part, "ARM" }, @@ -276,7 +300,7 @@ static const hw_impl hw_implementer[] = { { 0x44, dec_part, "DEC" }, { 0x46, fujitsu_part, "FUJITSU" }, { 0x48, hisi_part, "HiSilicon" }, - { 0x4e, nvidia_part, "Nvidia" }, + { 0x4e, nvidia_part, "NVIDIA" }, { 0x50, apm_part, "APM" }, { 0x51, qcom_part, "Qualcomm" }, { 0x53, samsung_part, "Samsung" }, @@ -284,6 +308,7 @@ static const hw_impl hw_implementer[] = { { 0x61, apple_part, "Apple" }, { 0x66, faraday_part, "Faraday" }, { 0x69, intel_part, "Intel" }, + { 0x6d, ms_part, "Microsoft" }, { 0x70, ft_part, "Phytium" }, { 0xc0, ampere_part, "Ampere" } }; diff --git a/src/backend/cuda/CudaBackend.cpp b/src/backend/cuda/CudaBackend.cpp index 5a732ae1..621563e1 100644 --- a/src/backend/cuda/CudaBackend.cpp +++ b/src/backend/cuda/CudaBackend.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2024 SChernykh + * Copyright (c) 2016-2024 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -227,7 +227,7 @@ public: # endif Log::print("|" CYAN_BOLD("%3zu") " |" CYAN_BOLD("%4u") " |" YELLOW(" %7s") " |" CYAN_BOLD("%10d") " |" CYAN_BOLD("%8d") " |" - CYAN_BOLD("%7d") " |" CYAN_BOLD("%3d") " |" CYAN_BOLD("%4d") " |" CYAN("%7zu") " | " GREEN("%s"), + CYAN_BOLD("%7d") " |" CYAN_BOLD("%3d") " |" CYAN_BOLD("%4d") " |" CYAN("%7zu") " | " GREEN_BOLD("%s"), i, data.thread.index(), data.device.topology().toString().data(), @@ -372,15 +372,20 @@ void xmrig::CudaBackend::printHashrate(bool details) char num[16 * 3] = { 0 }; - const double hashrate_short = hashrate()->calc(Hashrate::ShortInterval); - const double hashrate_medium = hashrate()->calc(Hashrate::MediumInterval); - const double hashrate_large = hashrate()->calc(Hashrate::LargeInterval); + auto hashrate_short = hashrate()->calc(Hashrate::ShortInterval); + auto hashrate_medium = hashrate()->calc(Hashrate::MediumInterval); + auto hashrate_large = hashrate()->calc(Hashrate::LargeInterval); double scale = 1.0; const char* h = " H/s"; - if ((hashrate_short >= 1e6) || (hashrate_medium >= 1e6) || (hashrate_large >= 1e6)) { + if ((hashrate_short.second >= 1e6) || (hashrate_medium.second >= 1e6) || (hashrate_large.second >= 1e6)) { scale = 1e-6; + + hashrate_short.second *= scale; + hashrate_medium.second *= scale; + hashrate_large.second *= scale; + h = "MH/s"; } @@ -388,12 +393,20 @@ void xmrig::CudaBackend::printHashrate(bool details) size_t i = 0; for (const auto& data : d_ptr->threads) { - Log::print("| %8zu | %8" PRId64 " | %8s | %8s | %8s |" CYAN_BOLD(" #%u") YELLOW(" %s") GREEN(" %s"), + auto h0 = hashrate()->calc(i, Hashrate::ShortInterval); + auto h1 = hashrate()->calc(i, Hashrate::MediumInterval); + auto h2 = hashrate()->calc(i, Hashrate::LargeInterval); + + h0.second *= scale; + h1.second *= scale; + h2.second *= scale; + + Log::print("| %8zu | %8" PRId64 " | %8s | %8s | %8s |" CYAN_BOLD(" #%u") YELLOW(" %s") GREEN(" %s"), i, data.thread.affinity(), - Hashrate::format(hashrate()->calc(i, Hashrate::ShortInterval) * scale, num, sizeof num / 3), - Hashrate::format(hashrate()->calc(i, Hashrate::MediumInterval) * scale, num + 16, sizeof num / 3), - Hashrate::format(hashrate()->calc(i, Hashrate::LargeInterval) * scale, num + 16 * 2, sizeof num / 3), + Hashrate::format(h0, num, sizeof num / 3), + Hashrate::format(h1, num + 16, sizeof num / 3), + Hashrate::format(h2, num + 16 * 2, sizeof num / 3), data.device.index(), data.device.topology().toString().data(), data.device.name().data() @@ -403,9 +416,9 @@ void xmrig::CudaBackend::printHashrate(bool details) } Log::print(WHITE_BOLD_S "| - | - | %8s | %8s | %8s |", - Hashrate::format(hashrate_short * scale, num, sizeof num / 3), - Hashrate::format(hashrate_medium * scale, num + 16, sizeof num / 3), - Hashrate::format(hashrate_large * scale, num + 16 * 2, sizeof num / 3) + Hashrate::format(hashrate_short , num, sizeof num / 3), + Hashrate::format(hashrate_medium, num + 16, sizeof num / 3), + Hashrate::format(hashrate_large , num + 16 * 2, sizeof num / 3) ); } diff --git a/src/backend/cuda/CudaWorker.cpp b/src/backend/cuda/CudaWorker.cpp index 97553637..a3fd11ef 100644 --- a/src/backend/cuda/CudaWorker.cpp +++ b/src/backend/cuda/CudaWorker.cpp @@ -158,7 +158,7 @@ void xmrig::CudaWorker::start() std::this_thread::yield(); } - if (!consumeJob()) { + if (isReady() && !consumeJob()) { return; } } diff --git a/src/backend/cuda/runners/CudaRxRunner.cpp b/src/backend/cuda/runners/CudaRxRunner.cpp index dd64e865..2ade7d26 100644 --- a/src/backend/cuda/runners/CudaRxRunner.cpp +++ b/src/backend/cuda/runners/CudaRxRunner.cpp @@ -5,8 +5,8 @@ * Copyright 2014-2016 Wolf9466 * Copyright 2016 Jay D Dee * Copyright 2017-2018 XMR-Stak , - * Copyright 2018-2019 SChernykh - * Copyright 2016-2019 XMRig , + * Copyright 2018-2024 SChernykh + * Copyright 2016-2024 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -22,7 +22,6 @@ * along with this program. If not, see . */ - #include "backend/cuda/runners/CudaRxRunner.h" #include "backend/cuda/CudaLaunchData.h" #include "backend/cuda/wrappers/CudaLib.h" @@ -55,12 +54,21 @@ bool xmrig::CudaRxRunner::run(uint32_t startNonce, uint32_t *rescount, uint32_t bool xmrig::CudaRxRunner::set(const Job &job, uint8_t *blob) { + if (!m_datasetHost && (m_seed != job.seed())) { + m_seed = job.seed(); + + if (m_ready) { + const auto *dataset = Rx::dataset(job, 0); + callWrapper(CudaLib::rxUpdateDataset(m_ctx, dataset->raw(), dataset->size(false))); + } + } + const bool rc = CudaBaseRunner::set(job, blob); if (!rc || m_ready) { return rc; } - auto dataset = Rx::dataset(job, 0); + const auto *dataset = Rx::dataset(job, 0); m_ready = callWrapper(CudaLib::rxPrepare(m_ctx, dataset->raw(), dataset->size(false), m_datasetHost, m_intensity)); return m_ready; diff --git a/src/backend/cuda/runners/CudaRxRunner.h b/src/backend/cuda/runners/CudaRxRunner.h index 448400bc..12e90783 100644 --- a/src/backend/cuda/runners/CudaRxRunner.h +++ b/src/backend/cuda/runners/CudaRxRunner.h @@ -5,8 +5,8 @@ * Copyright 2014-2016 Wolf9466 * Copyright 2016 Jay D Dee * Copyright 2017-2018 XMR-Stak , - * Copyright 2018-2019 SChernykh - * Copyright 2016-2019 XMRig , + * Copyright 2018-2024 SChernykh + * Copyright 2016-2024 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -27,6 +27,7 @@ #include "backend/cuda/runners/CudaBaseRunner.h" +#include "base/tools/Buffer.h" namespace xmrig { @@ -46,6 +47,7 @@ protected: private: bool m_ready = false; const bool m_datasetHost = false; + Buffer m_seed; size_t m_intensity = 0; }; diff --git a/src/backend/cuda/wrappers/CudaDevice.cpp b/src/backend/cuda/wrappers/CudaDevice.cpp index f06fe940..b160e428 100644 --- a/src/backend/cuda/wrappers/CudaDevice.cpp +++ b/src/backend/cuda/wrappers/CudaDevice.cpp @@ -5,8 +5,8 @@ * Copyright 2014-2016 Wolf9466 * Copyright 2016 Jay D Dee * Copyright 2017-2018 XMR-Stak , - * Copyright 2018-2020 SChernykh - * Copyright 2016-2020 XMRig , + * Copyright 2018-2024 SChernykh + * Copyright 2016-2024 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -22,7 +22,6 @@ * along with this program. If not, see . */ - #include "backend/cuda/wrappers/CudaDevice.h" #include "3rdparty/rapidjson/document.h" #include "backend/cuda/CudaThreads.h" @@ -41,7 +40,7 @@ xmrig::CudaDevice::CudaDevice(uint32_t index, int32_t bfactor, int32_t bsleep) : m_index(index) { - auto ctx = CudaLib::alloc(index, bfactor, bsleep); + auto *ctx = CudaLib::alloc(index, bfactor, bsleep); if (!CudaLib::deviceInfo(ctx, 0, 0, Algorithm::INVALID)) { CudaLib::release(ctx); @@ -50,7 +49,7 @@ xmrig::CudaDevice::CudaDevice(uint32_t index, int32_t bfactor, int32_t bsleep) : m_ctx = ctx; m_name = CudaLib::deviceName(ctx); - m_topology = PciTopology(CudaLib::deviceUint(ctx, CudaLib::DevicePciBusID), CudaLib::deviceUint(ctx, CudaLib::DevicePciDeviceID), 0); + m_topology = { CudaLib::deviceUint(ctx, CudaLib::DevicePciBusID), CudaLib::deviceUint(ctx, CudaLib::DevicePciDeviceID), 0U }; } diff --git a/src/backend/cuda/wrappers/CudaLib.cpp b/src/backend/cuda/wrappers/CudaLib.cpp index 8ca11fa5..87501f41 100644 --- a/src/backend/cuda/wrappers/CudaLib.cpp +++ b/src/backend/cuda/wrappers/CudaLib.cpp @@ -19,10 +19,10 @@ #include #include - #include "backend/cuda/wrappers/CudaLib.h" #include "base/io/Env.h" #include "base/io/log/Log.h" +#include "base/io/log/Tags.h" #include "base/kernel/Process.h" #include "crypto/rx/RxAlgo.h" @@ -68,6 +68,7 @@ static const char *kPluginVersion = "pluginVersion"; static const char *kRelease = "release"; static const char *kRxHash = "rxHash"; static const char *kRxPrepare = "rxPrepare"; +static const char *kRxUpdateDataset = "rxUpdateDataset"; static const char *kSetJob = "setJob"; static const char *kSetJob_v2 = "setJob_v2"; static const char *kVersion = "version"; @@ -92,6 +93,7 @@ using pluginVersion_t = const char * (*)(); using release_t = void (*)(nvid_ctx *); using rxHash_t = bool (*)(nvid_ctx *, uint32_t, uint64_t, uint32_t *, uint32_t *); using rxPrepare_t = bool (*)(nvid_ctx *, const void *, size_t, bool, uint32_t); +using rxUpdateDataset_t = bool (*)(nvid_ctx *, const void *, size_t); using setJob_t = bool (*)(nvid_ctx *, const void *, size_t, uint32_t); using setJob_v2_t = bool (*)(nvid_ctx *, const void *, size_t, const char *); using version_t = uint32_t (*)(Version); @@ -116,6 +118,7 @@ static pluginVersion_t pPluginVersion = nullptr; static release_t pRelease = nullptr; static rxHash_t pRxHash = nullptr; static rxPrepare_t pRxPrepare = nullptr; +static rxUpdateDataset_t pRxUpdateDataset = nullptr; static setJob_t pSetJob = nullptr; static setJob_v2_t pSetJob_v2 = nullptr; static version_t pVersion = nullptr; @@ -202,10 +205,26 @@ bool xmrig::CudaLib::rxHash(nvid_ctx *ctx, uint32_t startNonce, uint64_t target, bool xmrig::CudaLib::rxPrepare(nvid_ctx *ctx, const void *dataset, size_t datasetSize, bool dataset_host, uint32_t batchSize) noexcept { +# ifdef XMRIG_ALGO_RANDOMX + if (!pRxUpdateDataset) { + LOG_WARN("%s" YELLOW_BOLD("CUDA plugin is outdated. Please update to the latest version"), Tags::randomx()); + } +# endif + return pRxPrepare(ctx, dataset, datasetSize, dataset_host, batchSize); } +bool xmrig::CudaLib::rxUpdateDataset(nvid_ctx *ctx, const void *dataset, size_t datasetSize) noexcept +{ + if (pRxUpdateDataset) { + return pRxUpdateDataset(ctx, dataset, datasetSize); + } + + return true; +} + + bool xmrig::CudaLib::kawPowHash(nvid_ctx *ctx, uint8_t* job_blob, uint64_t target, uint32_t *rescount, uint32_t *resnonce, uint32_t *skipped_hashes) noexcept { return pKawPowHash(ctx, job_blob, target, rescount, resnonce, skipped_hashes); @@ -401,5 +420,7 @@ void xmrig::CudaLib::load() DLSYM(SetJob_v2); } + uv_dlsym(&cudaLib, kRxUpdateDataset, reinterpret_cast(&pRxUpdateDataset)); + pInit(); } diff --git a/src/backend/cuda/wrappers/CudaLib.h b/src/backend/cuda/wrappers/CudaLib.h index cf1a538b..ff7533ba 100644 --- a/src/backend/cuda/wrappers/CudaLib.h +++ b/src/backend/cuda/wrappers/CudaLib.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2024 SChernykh + * Copyright (c) 2016-2024 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -71,6 +71,7 @@ public: static bool deviceInit(nvid_ctx *ctx) noexcept; static bool rxHash(nvid_ctx *ctx, uint32_t startNonce, uint64_t target, uint32_t *rescount, uint32_t *resnonce) noexcept; static bool rxPrepare(nvid_ctx *ctx, const void *dataset, size_t datasetSize, bool dataset_host, uint32_t batchSize) noexcept; + static bool rxUpdateDataset(nvid_ctx *ctx, const void *dataset, size_t datasetSize) noexcept; static bool kawPowHash(nvid_ctx *ctx, uint8_t* job_blob, uint64_t target, uint32_t *rescount, uint32_t *resnonce, uint32_t *skipped_hashes) noexcept; static bool kawPowPrepare(nvid_ctx *ctx, const void* cache, size_t cache_size, const void* dag_precalc, size_t dag_size, uint32_t height, const uint64_t* dag_sizes) noexcept; static bool kawPowStopHash(nvid_ctx *ctx) noexcept; diff --git a/src/backend/opencl/OclBackend.cpp b/src/backend/opencl/OclBackend.cpp index 493ad4c8..820c63d4 100644 --- a/src/backend/opencl/OclBackend.cpp +++ b/src/backend/opencl/OclBackend.cpp @@ -352,15 +352,20 @@ void xmrig::OclBackend::printHashrate(bool details) char num[16 * 3] = { 0 }; - const double hashrate_short = hashrate()->calc(Hashrate::ShortInterval); - const double hashrate_medium = hashrate()->calc(Hashrate::MediumInterval); - const double hashrate_large = hashrate()->calc(Hashrate::LargeInterval); + auto hashrate_short = hashrate()->calc(Hashrate::ShortInterval); + auto hashrate_medium = hashrate()->calc(Hashrate::MediumInterval); + auto hashrate_large = hashrate()->calc(Hashrate::LargeInterval); double scale = 1.0; const char* h = " H/s"; - if ((hashrate_short >= 1e6) || (hashrate_medium >= 1e6) || (hashrate_large >= 1e6)) { + if ((hashrate_short.second >= 1e6) || (hashrate_medium.second >= 1e6) || (hashrate_large.second >= 1e6)) { scale = 1e-6; + + hashrate_short.second *= scale; + hashrate_medium.second *= scale; + hashrate_large.second *= scale; + h = "MH/s"; } @@ -368,12 +373,16 @@ void xmrig::OclBackend::printHashrate(bool details) size_t i = 0; for (const auto& data : d_ptr->threads) { - Log::print("| %8zu | %8" PRId64 " | %8s | %8s | %8s |" CYAN_BOLD(" #%u") YELLOW(" %s") " %s", + auto h0 = hashrate()->calc(i, Hashrate::ShortInterval); + auto h1 = hashrate()->calc(i, Hashrate::MediumInterval); + auto h2 = hashrate()->calc(i, Hashrate::LargeInterval); + + Log::print("| %8zu | %8" PRId64 " | %8s | %8s | %8s |" CYAN_BOLD(" #%u") YELLOW(" %s") " %s", i, data.affinity, - Hashrate::format(hashrate()->calc(i, Hashrate::ShortInterval) * scale, num, sizeof num / 3), - Hashrate::format(hashrate()->calc(i, Hashrate::MediumInterval) * scale, num + 16, sizeof num / 3), - Hashrate::format(hashrate()->calc(i, Hashrate::LargeInterval) * scale, num + 16 * 2, sizeof num / 3), + Hashrate::format(h0, num, sizeof num / 3), + Hashrate::format(h1, num + 16, sizeof num / 3), + Hashrate::format(h2, num + 16 * 2, sizeof num / 3), data.device.index(), data.device.topology().toString().data(), data.device.printableName().data() @@ -383,9 +392,9 @@ void xmrig::OclBackend::printHashrate(bool details) } Log::print(WHITE_BOLD_S "| - | - | %8s | %8s | %8s |", - Hashrate::format(hashrate_short * scale, num, sizeof num / 3), - Hashrate::format(hashrate_medium * scale, num + 16, sizeof num / 3), - Hashrate::format(hashrate_large * scale, num + 16 * 2, sizeof num / 3) + Hashrate::format(hashrate_short , num, sizeof num / 3), + Hashrate::format(hashrate_medium, num + 16, sizeof num / 3), + Hashrate::format(hashrate_large , num + 16 * 2, sizeof num / 3) ); } diff --git a/src/backend/opencl/OclWorker.cpp b/src/backend/opencl/OclWorker.cpp index 45b92863..dcac60b7 100644 --- a/src/backend/opencl/OclWorker.cpp +++ b/src/backend/opencl/OclWorker.cpp @@ -190,7 +190,7 @@ void xmrig::OclWorker::start() std::this_thread::yield(); } - if (!consumeJob()) { + if (isReady() && !consumeJob()) { return; } } diff --git a/src/backend/opencl/cl/kawpow/kawpow.cl b/src/backend/opencl/cl/kawpow/kawpow.cl index c1a6cb2f..8f914e8b 100644 --- a/src/backend/opencl/cl/kawpow/kawpow.cl +++ b/src/backend/opencl/cl/kawpow/kawpow.cl @@ -74,7 +74,7 @@ void keccak_f800_round(uint32_t st[25], const int r) // Keccak - implemented as a variant of SHAKE // The width is 800, with a bitrate of 576, a capacity of 224, and no padding // Only need 64 bits of output for mining -uint64_t keccak_f800(uint32_t* st) +void keccak_f800(uint32_t* st) { // Complete all 22 rounds as a separate impl to // evaluate only first 8 words is wasteful of regsters diff --git a/src/backend/opencl/cl/kawpow/kawpow_cl.h b/src/backend/opencl/cl/kawpow/kawpow_cl.h index e5ae5788..f514e05d 100644 --- a/src/backend/opencl/cl/kawpow/kawpow_cl.h +++ b/src/backend/opencl/cl/kawpow/kawpow_cl.h @@ -2,7 +2,7 @@ namespace xmrig { -static const char kawpow_cl[5948] = { +static const char kawpow_cl[5944] = { 0x23,0x69,0x66,0x64,0x65,0x66,0x20,0x63,0x6c,0x5f,0x63,0x6c,0x61,0x6e,0x67,0x5f,0x73,0x74,0x6f,0x72,0x61,0x67,0x65,0x5f,0x63,0x6c,0x61,0x73,0x73,0x5f,0x73,0x70, 0x65,0x63,0x69,0x66,0x69,0x65,0x72,0x73,0x0a,0x23,0x70,0x72,0x61,0x67,0x6d,0x61,0x20,0x4f,0x50,0x45,0x4e,0x43,0x4c,0x20,0x45,0x58,0x54,0x45,0x4e,0x53,0x49,0x4f, 0x4e,0x20,0x63,0x6c,0x5f,0x63,0x6c,0x61,0x6e,0x67,0x5f,0x73,0x74,0x6f,0x72,0x61,0x67,0x65,0x5f,0x63,0x6c,0x61,0x73,0x73,0x5f,0x73,0x70,0x65,0x63,0x69,0x66,0x69, @@ -76,119 +76,119 @@ static const char kawpow_cl[5948] = { 0x3d,0x73,0x74,0x5b,0x6a,0x2b,0x69,0x5d,0x3b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x35,0x3b,0x20,0x69,0x2b,0x2b, 0x29,0x0a,0x73,0x74,0x5b,0x6a,0x2b,0x69,0x5d,0x20,0x5e,0x3d,0x20,0x28,0x7e,0x62,0x63,0x5b,0x28,0x69,0x2b,0x31,0x29,0x20,0x25,0x20,0x35,0x5d,0x29,0x26,0x62,0x63, 0x5b,0x28,0x69,0x2b,0x32,0x29,0x20,0x25,0x20,0x35,0x5d,0x3b,0x0a,0x7d,0x0a,0x73,0x74,0x5b,0x30,0x5d,0x20,0x5e,0x3d,0x20,0x6b,0x65,0x63,0x63,0x61,0x6b,0x66,0x5f, - 0x72,0x6e,0x64,0x63,0x5b,0x72,0x5d,0x3b,0x0a,0x7d,0x0a,0x75,0x69,0x6e,0x74,0x36,0x34,0x5f,0x74,0x20,0x6b,0x65,0x63,0x63,0x61,0x6b,0x5f,0x66,0x38,0x30,0x30,0x28, - 0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x2a,0x20,0x73,0x74,0x29,0x0a,0x7b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x72,0x3d,0x30,0x3b,0x20,0x72,0x3c, - 0x32,0x32,0x3b,0x20,0x72,0x2b,0x2b,0x29,0x20,0x7b,0x0a,0x6b,0x65,0x63,0x63,0x61,0x6b,0x5f,0x66,0x38,0x30,0x30,0x5f,0x72,0x6f,0x75,0x6e,0x64,0x28,0x73,0x74,0x2c, - 0x72,0x29,0x3b,0x0a,0x7d,0x0a,0x7d,0x0a,0x23,0x64,0x65,0x66,0x69,0x6e,0x65,0x20,0x66,0x6e,0x76,0x31,0x61,0x28,0x68,0x2c,0x20,0x64,0x29,0x20,0x28,0x68,0x20,0x3d, - 0x20,0x28,0x68,0x20,0x5e,0x20,0x64,0x29,0x20,0x2a,0x20,0x46,0x4e,0x56,0x5f,0x50,0x52,0x49,0x4d,0x45,0x29,0x0a,0x74,0x79,0x70,0x65,0x64,0x65,0x66,0x20,0x73,0x74, - 0x72,0x75,0x63,0x74,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x7a,0x2c,0x77,0x2c,0x6a,0x73,0x72,0x2c,0x6a,0x63,0x6f,0x6e,0x67,0x3b,0x0a,0x7d, - 0x20,0x6b,0x69,0x73,0x73,0x39,0x39,0x5f,0x74,0x3b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6b,0x69,0x73,0x73,0x39,0x39,0x28,0x6b,0x69,0x73,0x73,0x39, - 0x39,0x5f,0x74,0x2a,0x20,0x73,0x74,0x29,0x0a,0x7b,0x0a,0x73,0x74,0x2d,0x3e,0x7a,0x3d,0x33,0x36,0x39,0x36,0x39,0x2a,0x28,0x73,0x74,0x2d,0x3e,0x7a,0x26,0x36,0x35, - 0x35,0x33,0x35,0x29,0x2b,0x28,0x73,0x74,0x2d,0x3e,0x7a,0x3e,0x3e,0x31,0x36,0x29,0x3b,0x0a,0x73,0x74,0x2d,0x3e,0x77,0x3d,0x31,0x38,0x30,0x30,0x30,0x2a,0x28,0x73, - 0x74,0x2d,0x3e,0x77,0x26,0x36,0x35,0x35,0x33,0x35,0x29,0x2b,0x28,0x73,0x74,0x2d,0x3e,0x77,0x3e,0x3e,0x31,0x36,0x29,0x3b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f, - 0x74,0x20,0x4d,0x57,0x43,0x3d,0x28,0x28,0x73,0x74,0x2d,0x3e,0x7a,0x3c,0x3c,0x31,0x36,0x29,0x2b,0x73,0x74,0x2d,0x3e,0x77,0x29,0x3b,0x0a,0x73,0x74,0x2d,0x3e,0x6a, - 0x73,0x72,0x20,0x5e,0x3d,0x20,0x28,0x73,0x74,0x2d,0x3e,0x6a,0x73,0x72,0x3c,0x3c,0x31,0x37,0x29,0x3b,0x0a,0x73,0x74,0x2d,0x3e,0x6a,0x73,0x72,0x20,0x5e,0x3d,0x20, - 0x28,0x73,0x74,0x2d,0x3e,0x6a,0x73,0x72,0x3e,0x3e,0x31,0x33,0x29,0x3b,0x0a,0x73,0x74,0x2d,0x3e,0x6a,0x73,0x72,0x20,0x5e,0x3d,0x20,0x28,0x73,0x74,0x2d,0x3e,0x6a, - 0x73,0x72,0x3c,0x3c,0x35,0x29,0x3b,0x0a,0x73,0x74,0x2d,0x3e,0x6a,0x63,0x6f,0x6e,0x67,0x3d,0x36,0x39,0x30,0x36,0x39,0x2a,0x73,0x74,0x2d,0x3e,0x6a,0x63,0x6f,0x6e, - 0x67,0x2b,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x3b,0x0a,0x72,0x65,0x74,0x75,0x72,0x6e,0x20,0x28,0x28,0x4d,0x57,0x43,0x5e,0x73,0x74,0x2d,0x3e,0x6a,0x63,0x6f,0x6e, - 0x67,0x29,0x2b,0x73,0x74,0x2d,0x3e,0x6a,0x73,0x72,0x29,0x3b,0x0a,0x7d,0x0a,0x76,0x6f,0x69,0x64,0x20,0x66,0x69,0x6c,0x6c,0x5f,0x6d,0x69,0x78,0x28,0x6c,0x6f,0x63, - 0x61,0x6c,0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x2a,0x20,0x73,0x65,0x65,0x64,0x2c,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6c,0x61,0x6e,0x65,0x5f, - 0x69,0x64,0x2c,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x2a,0x20,0x6d,0x69,0x78,0x29,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x66,0x6e,0x76, - 0x5f,0x68,0x61,0x73,0x68,0x3d,0x46,0x4e,0x56,0x5f,0x4f,0x46,0x46,0x53,0x45,0x54,0x5f,0x42,0x41,0x53,0x49,0x53,0x3b,0x0a,0x6b,0x69,0x73,0x73,0x39,0x39,0x5f,0x74, - 0x20,0x73,0x74,0x3b,0x0a,0x73,0x74,0x2e,0x7a,0x3d,0x66,0x6e,0x76,0x31,0x61,0x28,0x66,0x6e,0x76,0x5f,0x68,0x61,0x73,0x68,0x2c,0x73,0x65,0x65,0x64,0x5b,0x30,0x5d, - 0x29,0x3b,0x0a,0x73,0x74,0x2e,0x77,0x3d,0x66,0x6e,0x76,0x31,0x61,0x28,0x66,0x6e,0x76,0x5f,0x68,0x61,0x73,0x68,0x2c,0x73,0x65,0x65,0x64,0x5b,0x31,0x5d,0x29,0x3b, - 0x0a,0x73,0x74,0x2e,0x6a,0x73,0x72,0x3d,0x66,0x6e,0x76,0x31,0x61,0x28,0x66,0x6e,0x76,0x5f,0x68,0x61,0x73,0x68,0x2c,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x29,0x3b, - 0x0a,0x73,0x74,0x2e,0x6a,0x63,0x6f,0x6e,0x67,0x3d,0x66,0x6e,0x76,0x31,0x61,0x28,0x66,0x6e,0x76,0x5f,0x68,0x61,0x73,0x68,0x2c,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64, - 0x29,0x3b,0x0a,0x23,0x70,0x72,0x61,0x67,0x6d,0x61,0x20,0x75,0x6e,0x72,0x6f,0x6c,0x6c,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20, - 0x69,0x3c,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x52,0x45,0x47,0x53,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x6d,0x69,0x78,0x5b,0x69,0x5d,0x3d,0x6b,0x69,0x73,0x73, - 0x39,0x39,0x28,0x26,0x73,0x74,0x29,0x3b,0x0a,0x7d,0x0a,0x74,0x79,0x70,0x65,0x64,0x65,0x66,0x20,0x73,0x74,0x72,0x75,0x63,0x74,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74, - 0x33,0x32,0x5f,0x74,0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e,0x45,0x53,0x5d,0x3b,0x0a,0x7d,0x20,0x73, - 0x68,0x75,0x66,0x66,0x6c,0x65,0x5f,0x74,0x3b,0x0a,0x74,0x79,0x70,0x65,0x64,0x65,0x66,0x20,0x73,0x74,0x72,0x75,0x63,0x74,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33, - 0x32,0x5f,0x74,0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x33,0x32,0x2f,0x73,0x69,0x7a,0x65,0x6f,0x66,0x28,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x29,0x5d, - 0x3b,0x0a,0x7d,0x20,0x68,0x61,0x73,0x68,0x33,0x32,0x5f,0x74,0x3b,0x0a,0x23,0x69,0x66,0x20,0x50,0x4c,0x41,0x54,0x46,0x4f,0x52,0x4d,0x20,0x21,0x3d,0x20,0x4f,0x50, - 0x45,0x4e,0x43,0x4c,0x5f,0x50,0x4c,0x41,0x54,0x46,0x4f,0x52,0x4d,0x5f,0x4e,0x56,0x49,0x44,0x49,0x41,0x20,0x0a,0x5f,0x5f,0x61,0x74,0x74,0x72,0x69,0x62,0x75,0x74, - 0x65,0x5f,0x5f,0x28,0x28,0x72,0x65,0x71,0x64,0x5f,0x77,0x6f,0x72,0x6b,0x5f,0x67,0x72,0x6f,0x75,0x70,0x5f,0x73,0x69,0x7a,0x65,0x28,0x47,0x52,0x4f,0x55,0x50,0x5f, - 0x53,0x49,0x5a,0x45,0x2c,0x31,0x2c,0x31,0x29,0x29,0x29,0x0a,0x23,0x65,0x6e,0x64,0x69,0x66,0x0a,0x5f,0x5f,0x6b,0x65,0x72,0x6e,0x65,0x6c,0x20,0x76,0x6f,0x69,0x64, - 0x20,0x70,0x72,0x6f,0x67,0x70,0x6f,0x77,0x5f,0x73,0x65,0x61,0x72,0x63,0x68,0x28,0x5f,0x5f,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x20,0x64,0x61,0x67,0x5f,0x74,0x20,0x63, - 0x6f,0x6e,0x73,0x74,0x2a,0x20,0x67,0x5f,0x64,0x61,0x67,0x2c,0x5f,0x5f,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x20,0x75,0x69,0x6e,0x74,0x2a,0x20,0x6a,0x6f,0x62,0x5f,0x62, - 0x6c,0x6f,0x62,0x2c,0x75,0x6c,0x6f,0x6e,0x67,0x20,0x74,0x61,0x72,0x67,0x65,0x74,0x2c,0x75,0x69,0x6e,0x74,0x20,0x68,0x61,0x63,0x6b,0x5f,0x66,0x61,0x6c,0x73,0x65, - 0x2c,0x76,0x6f,0x6c,0x61,0x74,0x69,0x6c,0x65,0x20,0x5f,0x5f,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x20,0x75,0x69,0x6e,0x74,0x2a,0x20,0x72,0x65,0x73,0x75,0x6c,0x74,0x73, - 0x2c,0x76,0x6f,0x6c,0x61,0x74,0x69,0x6c,0x65,0x20,0x5f,0x5f,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x20,0x75,0x69,0x6e,0x74,0x2a,0x20,0x73,0x74,0x6f,0x70,0x29,0x0a,0x7b, - 0x0a,0x63,0x6f,0x6e,0x73,0x74,0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6c,0x69,0x64,0x3d,0x67,0x65,0x74,0x5f,0x6c,0x6f,0x63,0x61,0x6c,0x5f,0x69,0x64, - 0x28,0x30,0x29,0x3b,0x0a,0x63,0x6f,0x6e,0x73,0x74,0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x67,0x69,0x64,0x3d,0x67,0x65,0x74,0x5f,0x67,0x6c,0x6f,0x62, - 0x61,0x6c,0x5f,0x69,0x64,0x28,0x30,0x29,0x3b,0x0a,0x69,0x66,0x28,0x73,0x74,0x6f,0x70,0x5b,0x30,0x5d,0x29,0x20,0x7b,0x0a,0x69,0x66,0x28,0x6c,0x69,0x64,0x3d,0x3d, - 0x30,0x29,0x20,0x7b,0x0a,0x61,0x74,0x6f,0x6d,0x69,0x63,0x5f,0x69,0x6e,0x63,0x28,0x73,0x74,0x6f,0x70,0x2b,0x31,0x29,0x3b,0x0a,0x7d,0x0a,0x72,0x65,0x74,0x75,0x72, - 0x6e,0x3b,0x0a,0x7d,0x0a,0x5f,0x5f,0x6c,0x6f,0x63,0x61,0x6c,0x20,0x73,0x68,0x75,0x66,0x66,0x6c,0x65,0x5f,0x74,0x20,0x73,0x68,0x61,0x72,0x65,0x5b,0x48,0x41,0x53, - 0x48,0x45,0x53,0x5f,0x50,0x45,0x52,0x5f,0x47,0x52,0x4f,0x55,0x50,0x5d,0x3b,0x0a,0x5f,0x5f,0x6c,0x6f,0x63,0x61,0x6c,0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74, - 0x20,0x63,0x5f,0x64,0x61,0x67,0x5b,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x43,0x41,0x43,0x48,0x45,0x5f,0x57,0x4f,0x52,0x44,0x53,0x5d,0x3b,0x0a,0x63,0x6f,0x6e, - 0x73,0x74,0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x3d,0x6c,0x69,0x64,0x26,0x28,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57, - 0x5f,0x4c,0x41,0x4e,0x45,0x53,0x2d,0x31,0x29,0x3b,0x0a,0x63,0x6f,0x6e,0x73,0x74,0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x67,0x72,0x6f,0x75,0x70,0x5f, - 0x69,0x64,0x3d,0x6c,0x69,0x64,0x2f,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e,0x45,0x53,0x3b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x75,0x69,0x6e,0x74,0x33, - 0x32,0x5f,0x74,0x20,0x77,0x6f,0x72,0x64,0x3d,0x6c,0x69,0x64,0x2a,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x44,0x41,0x47,0x5f,0x4c,0x4f,0x41,0x44,0x53,0x3b,0x20, - 0x77,0x6f,0x72,0x64,0x3c,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x43,0x41,0x43,0x48,0x45,0x5f,0x57,0x4f,0x52,0x44,0x53,0x3b,0x20,0x77,0x6f,0x72,0x64,0x2b,0x3d, - 0x47,0x52,0x4f,0x55,0x50,0x5f,0x53,0x49,0x5a,0x45,0x2a,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x44,0x41,0x47,0x5f,0x4c,0x4f,0x41,0x44,0x53,0x29,0x0a,0x7b,0x0a, - 0x64,0x61,0x67,0x5f,0x74,0x20,0x6c,0x6f,0x61,0x64,0x3d,0x67,0x5f,0x64,0x61,0x67,0x5b,0x77,0x6f,0x72,0x64,0x2f,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x44,0x41, - 0x47,0x5f,0x4c,0x4f,0x41,0x44,0x53,0x5d,0x3b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x50,0x52,0x4f,0x47,0x50,0x4f, - 0x57,0x5f,0x44,0x41,0x47,0x5f,0x4c,0x4f,0x41,0x44,0x53,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x63,0x5f,0x64,0x61,0x67,0x5b,0x77,0x6f,0x72,0x64,0x2b,0x69,0x5d,0x3d, - 0x6c,0x6f,0x61,0x64,0x2e,0x73,0x5b,0x69,0x5d,0x3b,0x0a,0x7d,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x68,0x61,0x73,0x68,0x5f,0x73,0x65,0x65,0x64,0x5b, - 0x32,0x5d,0x3b,0x20,0x0a,0x68,0x61,0x73,0x68,0x33,0x32,0x5f,0x74,0x20,0x64,0x69,0x67,0x65,0x73,0x74,0x3b,0x20,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20, - 0x73,0x74,0x61,0x74,0x65,0x32,0x5b,0x38,0x5d,0x3b,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x73,0x74,0x61,0x74,0x65,0x5b,0x32,0x35,0x5d,0x3b, - 0x20,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x31,0x30,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x73,0x74,0x61,0x74,0x65, - 0x5b,0x69,0x5d,0x3d,0x6a,0x6f,0x62,0x5f,0x62,0x6c,0x6f,0x62,0x5b,0x69,0x5d,0x3b,0x0a,0x73,0x74,0x61,0x74,0x65,0x5b,0x38,0x5d,0x3d,0x67,0x69,0x64,0x3b,0x0a,0x66, - 0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x31,0x30,0x3b,0x20,0x69,0x3c,0x32,0x35,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x73,0x74,0x61,0x74,0x65,0x5b,0x69, - 0x5d,0x3d,0x72,0x61,0x76,0x65,0x6e,0x63,0x6f,0x69,0x6e,0x5f,0x72,0x6e,0x64,0x63,0x5b,0x69,0x2d,0x31,0x30,0x5d,0x3b,0x0a,0x6b,0x65,0x63,0x63,0x61,0x6b,0x5f,0x66, - 0x38,0x30,0x30,0x28,0x73,0x74,0x61,0x74,0x65,0x29,0x3b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x38,0x3b,0x20,0x69, - 0x2b,0x2b,0x29,0x0a,0x73,0x74,0x61,0x74,0x65,0x32,0x5b,0x69,0x5d,0x3d,0x73,0x74,0x61,0x74,0x65,0x5b,0x69,0x5d,0x3b,0x0a,0x7d,0x0a,0x23,0x70,0x72,0x61,0x67,0x6d, - 0x61,0x20,0x75,0x6e,0x72,0x6f,0x6c,0x6c,0x20,0x31,0x0a,0x66,0x6f,0x72,0x20,0x28,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x68,0x3d,0x30,0x3b,0x20,0x68,0x3c, - 0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e,0x45,0x53,0x3b,0x20,0x68,0x2b,0x2b,0x29,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6d, - 0x69,0x78,0x5b,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x52,0x45,0x47,0x53,0x5d,0x3b,0x0a,0x69,0x66,0x28,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x3d,0x3d,0x68,0x29, - 0x20,0x7b,0x0a,0x73,0x68,0x61,0x72,0x65,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f,0x69,0x64,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x30,0x5d,0x3d,0x73,0x74, - 0x61,0x74,0x65,0x32,0x5b,0x30,0x5d,0x3b,0x0a,0x73,0x68,0x61,0x72,0x65,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f,0x69,0x64,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73, - 0x5b,0x31,0x5d,0x3d,0x73,0x74,0x61,0x74,0x65,0x32,0x5b,0x31,0x5d,0x3b,0x0a,0x7d,0x0a,0x62,0x61,0x72,0x72,0x69,0x65,0x72,0x28,0x43,0x4c,0x4b,0x5f,0x4c,0x4f,0x43, - 0x41,0x4c,0x5f,0x4d,0x45,0x4d,0x5f,0x46,0x45,0x4e,0x43,0x45,0x29,0x3b,0x0a,0x66,0x69,0x6c,0x6c,0x5f,0x6d,0x69,0x78,0x28,0x73,0x68,0x61,0x72,0x65,0x5b,0x67,0x72, - 0x6f,0x75,0x70,0x5f,0x69,0x64,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x2c,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x2c,0x6d,0x69,0x78,0x29,0x3b,0x0a,0x23,0x70, - 0x72,0x61,0x67,0x6d,0x61,0x20,0x75,0x6e,0x72,0x6f,0x6c,0x6c,0x20,0x32,0x0a,0x66,0x6f,0x72,0x20,0x28,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6c,0x6f,0x6f, - 0x70,0x3d,0x30,0x3b,0x20,0x6c,0x6f,0x6f,0x70,0x3c,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x43,0x4e,0x54,0x5f,0x44,0x41,0x47,0x3b,0x20,0x2b,0x2b,0x6c,0x6f,0x6f, - 0x70,0x29,0x0a,0x7b,0x0a,0x69,0x66,0x28,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x3d,0x3d,0x28,0x6c,0x6f,0x6f,0x70,0x20,0x25,0x20,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57, - 0x5f,0x4c,0x41,0x4e,0x45,0x53,0x29,0x29,0x0a,0x73,0x68,0x61,0x72,0x65,0x5b,0x30,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f, - 0x69,0x64,0x5d,0x3d,0x6d,0x69,0x78,0x5b,0x30,0x5d,0x3b,0x0a,0x62,0x61,0x72,0x72,0x69,0x65,0x72,0x28,0x43,0x4c,0x4b,0x5f,0x4c,0x4f,0x43,0x41,0x4c,0x5f,0x4d,0x45, - 0x4d,0x5f,0x46,0x45,0x4e,0x43,0x45,0x29,0x3b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6f,0x66,0x66,0x73,0x65,0x74,0x3d,0x73,0x68,0x61,0x72,0x65,0x5b, - 0x30,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f,0x69,0x64,0x5d,0x3b,0x0a,0x6f,0x66,0x66,0x73,0x65,0x74,0x20,0x25,0x3d,0x20, - 0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x44,0x41,0x47,0x5f,0x45,0x4c,0x45,0x4d,0x45,0x4e,0x54,0x53,0x3b,0x0a,0x6f,0x66,0x66,0x73,0x65,0x74,0x3d,0x6f,0x66,0x66, - 0x73,0x65,0x74,0x2a,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e,0x45,0x53,0x2b,0x28,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x5e,0x6c,0x6f,0x6f,0x70,0x29, - 0x20,0x25,0x20,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e,0x45,0x53,0x3b,0x0a,0x64,0x61,0x67,0x5f,0x74,0x20,0x64,0x61,0x74,0x61,0x5f,0x64,0x61,0x67, - 0x3d,0x67,0x5f,0x64,0x61,0x67,0x5b,0x6f,0x66,0x66,0x73,0x65,0x74,0x5d,0x3b,0x0a,0x69,0x66,0x28,0x68,0x61,0x63,0x6b,0x5f,0x66,0x61,0x6c,0x73,0x65,0x29,0x20,0x62, - 0x61,0x72,0x72,0x69,0x65,0x72,0x28,0x43,0x4c,0x4b,0x5f,0x4c,0x4f,0x43,0x41,0x4c,0x5f,0x4d,0x45,0x4d,0x5f,0x46,0x45,0x4e,0x43,0x45,0x29,0x3b,0x0a,0x75,0x69,0x6e, - 0x74,0x33,0x32,0x5f,0x74,0x20,0x64,0x61,0x74,0x61,0x3b,0x0a,0x58,0x4d,0x52,0x49,0x47,0x5f,0x49,0x4e,0x43,0x4c,0x55,0x44,0x45,0x5f,0x50,0x52,0x4f,0x47,0x50,0x4f, - 0x57,0x5f,0x52,0x41,0x4e,0x44,0x4f,0x4d,0x5f,0x4d,0x41,0x54,0x48,0x0a,0x69,0x66,0x28,0x68,0x61,0x63,0x6b,0x5f,0x66,0x61,0x6c,0x73,0x65,0x29,0x20,0x62,0x61,0x72, - 0x72,0x69,0x65,0x72,0x28,0x43,0x4c,0x4b,0x5f,0x4c,0x4f,0x43,0x41,0x4c,0x5f,0x4d,0x45,0x4d,0x5f,0x46,0x45,0x4e,0x43,0x45,0x29,0x3b,0x0a,0x58,0x4d,0x52,0x49,0x47, - 0x5f,0x49,0x4e,0x43,0x4c,0x55,0x44,0x45,0x5f,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x44,0x41,0x54,0x41,0x5f,0x4c,0x4f,0x41,0x44,0x53,0x0a,0x7d,0x0a,0x75,0x69, - 0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6d,0x69,0x78,0x5f,0x68,0x61,0x73,0x68,0x3d,0x46,0x4e,0x56,0x5f,0x4f,0x46,0x46,0x53,0x45,0x54,0x5f,0x42,0x41,0x53,0x49,0x53, - 0x3b,0x0a,0x23,0x70,0x72,0x61,0x67,0x6d,0x61,0x20,0x75,0x6e,0x72,0x6f,0x6c,0x6c,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69, - 0x3c,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x52,0x45,0x47,0x53,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x66,0x6e,0x76,0x31,0x61,0x28,0x6d,0x69,0x78,0x5f,0x68,0x61, - 0x73,0x68,0x2c,0x6d,0x69,0x78,0x5b,0x69,0x5d,0x29,0x3b,0x0a,0x68,0x61,0x73,0x68,0x33,0x32,0x5f,0x74,0x20,0x64,0x69,0x67,0x65,0x73,0x74,0x5f,0x74,0x65,0x6d,0x70, - 0x3b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x38,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x64,0x69,0x67,0x65,0x73,0x74, - 0x5f,0x74,0x65,0x6d,0x70,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x69,0x5d,0x3d,0x46,0x4e,0x56,0x5f,0x4f,0x46,0x46,0x53,0x45,0x54,0x5f,0x42,0x41,0x53,0x49, - 0x53,0x3b,0x0a,0x73,0x68,0x61,0x72,0x65,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f,0x69,0x64,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x6c,0x61,0x6e,0x65,0x5f, - 0x69,0x64,0x5d,0x3d,0x6d,0x69,0x78,0x5f,0x68,0x61,0x73,0x68,0x3b,0x0a,0x62,0x61,0x72,0x72,0x69,0x65,0x72,0x28,0x43,0x4c,0x4b,0x5f,0x4c,0x4f,0x43,0x41,0x4c,0x5f, - 0x4d,0x45,0x4d,0x5f,0x46,0x45,0x4e,0x43,0x45,0x29,0x3b,0x0a,0x23,0x70,0x72,0x61,0x67,0x6d,0x61,0x20,0x75,0x6e,0x72,0x6f,0x6c,0x6c,0x0a,0x66,0x6f,0x72,0x20,0x28, - 0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e,0x45,0x53,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x66, - 0x6e,0x76,0x31,0x61,0x28,0x64,0x69,0x67,0x65,0x73,0x74,0x5f,0x74,0x65,0x6d,0x70,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x69,0x20,0x25,0x20,0x38,0x5d,0x2c, - 0x73,0x68,0x61,0x72,0x65,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f,0x69,0x64,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x69,0x5d,0x29,0x3b,0x0a,0x69,0x66,0x28, - 0x68,0x3d,0x3d,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x29,0x0a,0x64,0x69,0x67,0x65,0x73,0x74,0x3d,0x64,0x69,0x67,0x65,0x73,0x74,0x5f,0x74,0x65,0x6d,0x70,0x3b,0x0a, - 0x7d,0x0a,0x75,0x69,0x6e,0x74,0x36,0x34,0x5f,0x74,0x20,0x72,0x65,0x73,0x75,0x6c,0x74,0x3b,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x73,0x74, - 0x61,0x74,0x65,0x5b,0x32,0x35,0x5d,0x3d,0x7b,0x30,0x78,0x30,0x7d,0x3b,0x20,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c, - 0x38,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x73,0x74,0x61,0x74,0x65,0x5b,0x69,0x5d,0x3d,0x73,0x74,0x61,0x74,0x65,0x32,0x5b,0x69,0x5d,0x3b,0x0a,0x66,0x6f,0x72,0x20, - 0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x38,0x3b,0x20,0x69,0x3c,0x31,0x36,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x73,0x74,0x61,0x74,0x65,0x5b,0x69,0x5d,0x3d,0x64,0x69, - 0x67,0x65,0x73,0x74,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x69,0x2d,0x38,0x5d,0x3b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x31,0x36, - 0x3b,0x20,0x69,0x3c,0x32,0x35,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x73,0x74,0x61,0x74,0x65,0x5b,0x69,0x5d,0x3d,0x72,0x61,0x76,0x65,0x6e,0x63,0x6f,0x69,0x6e,0x5f, - 0x72,0x6e,0x64,0x63,0x5b,0x69,0x2d,0x31,0x36,0x5d,0x3b,0x0a,0x6b,0x65,0x63,0x63,0x61,0x6b,0x5f,0x66,0x38,0x30,0x30,0x28,0x73,0x74,0x61,0x74,0x65,0x29,0x3b,0x0a, - 0x75,0x69,0x6e,0x74,0x36,0x34,0x5f,0x74,0x20,0x72,0x65,0x73,0x3d,0x28,0x75,0x69,0x6e,0x74,0x36,0x34,0x5f,0x74,0x29,0x73,0x74,0x61,0x74,0x65,0x5b,0x31,0x5d,0x3c, - 0x3c,0x33,0x32,0x7c,0x73,0x74,0x61,0x74,0x65,0x5b,0x30,0x5d,0x3b,0x0a,0x72,0x65,0x73,0x75,0x6c,0x74,0x3d,0x61,0x73,0x5f,0x75,0x6c,0x6f,0x6e,0x67,0x28,0x61,0x73, - 0x5f,0x75,0x63,0x68,0x61,0x72,0x38,0x28,0x72,0x65,0x73,0x29,0x2e,0x73,0x37,0x36,0x35,0x34,0x33,0x32,0x31,0x30,0x29,0x3b,0x0a,0x7d,0x0a,0x69,0x66,0x28,0x72,0x65, - 0x73,0x75,0x6c,0x74,0x3c,0x3d,0x74,0x61,0x72,0x67,0x65,0x74,0x29,0x0a,0x7b,0x0a,0x2a,0x73,0x74,0x6f,0x70,0x3d,0x31,0x3b,0x0a,0x63,0x6f,0x6e,0x73,0x74,0x20,0x75, - 0x69,0x6e,0x74,0x20,0x6b,0x3d,0x61,0x74,0x6f,0x6d,0x69,0x63,0x5f,0x69,0x6e,0x63,0x28,0x72,0x65,0x73,0x75,0x6c,0x74,0x73,0x29,0x2b,0x31,0x3b,0x0a,0x69,0x66,0x28, - 0x6b,0x3c,0x3d,0x31,0x35,0x29,0x0a,0x72,0x65,0x73,0x75,0x6c,0x74,0x73,0x5b,0x6b,0x5d,0x3d,0x67,0x69,0x64,0x3b,0x0a,0x7d,0x0a,0x7d,0x0a,0x00 + 0x72,0x6e,0x64,0x63,0x5b,0x72,0x5d,0x3b,0x0a,0x7d,0x0a,0x76,0x6f,0x69,0x64,0x20,0x6b,0x65,0x63,0x63,0x61,0x6b,0x5f,0x66,0x38,0x30,0x30,0x28,0x75,0x69,0x6e,0x74, + 0x33,0x32,0x5f,0x74,0x2a,0x20,0x73,0x74,0x29,0x0a,0x7b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x72,0x3d,0x30,0x3b,0x20,0x72,0x3c,0x32,0x32,0x3b,0x20, + 0x72,0x2b,0x2b,0x29,0x20,0x7b,0x0a,0x6b,0x65,0x63,0x63,0x61,0x6b,0x5f,0x66,0x38,0x30,0x30,0x5f,0x72,0x6f,0x75,0x6e,0x64,0x28,0x73,0x74,0x2c,0x72,0x29,0x3b,0x0a, + 0x7d,0x0a,0x7d,0x0a,0x23,0x64,0x65,0x66,0x69,0x6e,0x65,0x20,0x66,0x6e,0x76,0x31,0x61,0x28,0x68,0x2c,0x20,0x64,0x29,0x20,0x28,0x68,0x20,0x3d,0x20,0x28,0x68,0x20, + 0x5e,0x20,0x64,0x29,0x20,0x2a,0x20,0x46,0x4e,0x56,0x5f,0x50,0x52,0x49,0x4d,0x45,0x29,0x0a,0x74,0x79,0x70,0x65,0x64,0x65,0x66,0x20,0x73,0x74,0x72,0x75,0x63,0x74, + 0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x7a,0x2c,0x77,0x2c,0x6a,0x73,0x72,0x2c,0x6a,0x63,0x6f,0x6e,0x67,0x3b,0x0a,0x7d,0x20,0x6b,0x69,0x73, + 0x73,0x39,0x39,0x5f,0x74,0x3b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6b,0x69,0x73,0x73,0x39,0x39,0x28,0x6b,0x69,0x73,0x73,0x39,0x39,0x5f,0x74,0x2a, + 0x20,0x73,0x74,0x29,0x0a,0x7b,0x0a,0x73,0x74,0x2d,0x3e,0x7a,0x3d,0x33,0x36,0x39,0x36,0x39,0x2a,0x28,0x73,0x74,0x2d,0x3e,0x7a,0x26,0x36,0x35,0x35,0x33,0x35,0x29, + 0x2b,0x28,0x73,0x74,0x2d,0x3e,0x7a,0x3e,0x3e,0x31,0x36,0x29,0x3b,0x0a,0x73,0x74,0x2d,0x3e,0x77,0x3d,0x31,0x38,0x30,0x30,0x30,0x2a,0x28,0x73,0x74,0x2d,0x3e,0x77, + 0x26,0x36,0x35,0x35,0x33,0x35,0x29,0x2b,0x28,0x73,0x74,0x2d,0x3e,0x77,0x3e,0x3e,0x31,0x36,0x29,0x3b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x4d,0x57, + 0x43,0x3d,0x28,0x28,0x73,0x74,0x2d,0x3e,0x7a,0x3c,0x3c,0x31,0x36,0x29,0x2b,0x73,0x74,0x2d,0x3e,0x77,0x29,0x3b,0x0a,0x73,0x74,0x2d,0x3e,0x6a,0x73,0x72,0x20,0x5e, + 0x3d,0x20,0x28,0x73,0x74,0x2d,0x3e,0x6a,0x73,0x72,0x3c,0x3c,0x31,0x37,0x29,0x3b,0x0a,0x73,0x74,0x2d,0x3e,0x6a,0x73,0x72,0x20,0x5e,0x3d,0x20,0x28,0x73,0x74,0x2d, + 0x3e,0x6a,0x73,0x72,0x3e,0x3e,0x31,0x33,0x29,0x3b,0x0a,0x73,0x74,0x2d,0x3e,0x6a,0x73,0x72,0x20,0x5e,0x3d,0x20,0x28,0x73,0x74,0x2d,0x3e,0x6a,0x73,0x72,0x3c,0x3c, + 0x35,0x29,0x3b,0x0a,0x73,0x74,0x2d,0x3e,0x6a,0x63,0x6f,0x6e,0x67,0x3d,0x36,0x39,0x30,0x36,0x39,0x2a,0x73,0x74,0x2d,0x3e,0x6a,0x63,0x6f,0x6e,0x67,0x2b,0x31,0x32, + 0x33,0x34,0x35,0x36,0x37,0x3b,0x0a,0x72,0x65,0x74,0x75,0x72,0x6e,0x20,0x28,0x28,0x4d,0x57,0x43,0x5e,0x73,0x74,0x2d,0x3e,0x6a,0x63,0x6f,0x6e,0x67,0x29,0x2b,0x73, + 0x74,0x2d,0x3e,0x6a,0x73,0x72,0x29,0x3b,0x0a,0x7d,0x0a,0x76,0x6f,0x69,0x64,0x20,0x66,0x69,0x6c,0x6c,0x5f,0x6d,0x69,0x78,0x28,0x6c,0x6f,0x63,0x61,0x6c,0x20,0x75, + 0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x2a,0x20,0x73,0x65,0x65,0x64,0x2c,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x2c,0x75, + 0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x2a,0x20,0x6d,0x69,0x78,0x29,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x66,0x6e,0x76,0x5f,0x68,0x61,0x73, + 0x68,0x3d,0x46,0x4e,0x56,0x5f,0x4f,0x46,0x46,0x53,0x45,0x54,0x5f,0x42,0x41,0x53,0x49,0x53,0x3b,0x0a,0x6b,0x69,0x73,0x73,0x39,0x39,0x5f,0x74,0x20,0x73,0x74,0x3b, + 0x0a,0x73,0x74,0x2e,0x7a,0x3d,0x66,0x6e,0x76,0x31,0x61,0x28,0x66,0x6e,0x76,0x5f,0x68,0x61,0x73,0x68,0x2c,0x73,0x65,0x65,0x64,0x5b,0x30,0x5d,0x29,0x3b,0x0a,0x73, + 0x74,0x2e,0x77,0x3d,0x66,0x6e,0x76,0x31,0x61,0x28,0x66,0x6e,0x76,0x5f,0x68,0x61,0x73,0x68,0x2c,0x73,0x65,0x65,0x64,0x5b,0x31,0x5d,0x29,0x3b,0x0a,0x73,0x74,0x2e, + 0x6a,0x73,0x72,0x3d,0x66,0x6e,0x76,0x31,0x61,0x28,0x66,0x6e,0x76,0x5f,0x68,0x61,0x73,0x68,0x2c,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x29,0x3b,0x0a,0x73,0x74,0x2e, + 0x6a,0x63,0x6f,0x6e,0x67,0x3d,0x66,0x6e,0x76,0x31,0x61,0x28,0x66,0x6e,0x76,0x5f,0x68,0x61,0x73,0x68,0x2c,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x29,0x3b,0x0a,0x23, + 0x70,0x72,0x61,0x67,0x6d,0x61,0x20,0x75,0x6e,0x72,0x6f,0x6c,0x6c,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x50,0x52, + 0x4f,0x47,0x50,0x4f,0x57,0x5f,0x52,0x45,0x47,0x53,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x6d,0x69,0x78,0x5b,0x69,0x5d,0x3d,0x6b,0x69,0x73,0x73,0x39,0x39,0x28,0x26, + 0x73,0x74,0x29,0x3b,0x0a,0x7d,0x0a,0x74,0x79,0x70,0x65,0x64,0x65,0x66,0x20,0x73,0x74,0x72,0x75,0x63,0x74,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74, + 0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e,0x45,0x53,0x5d,0x3b,0x0a,0x7d,0x20,0x73,0x68,0x75,0x66,0x66, + 0x6c,0x65,0x5f,0x74,0x3b,0x0a,0x74,0x79,0x70,0x65,0x64,0x65,0x66,0x20,0x73,0x74,0x72,0x75,0x63,0x74,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20, + 0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x33,0x32,0x2f,0x73,0x69,0x7a,0x65,0x6f,0x66,0x28,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x29,0x5d,0x3b,0x0a,0x7d,0x20, + 0x68,0x61,0x73,0x68,0x33,0x32,0x5f,0x74,0x3b,0x0a,0x23,0x69,0x66,0x20,0x50,0x4c,0x41,0x54,0x46,0x4f,0x52,0x4d,0x20,0x21,0x3d,0x20,0x4f,0x50,0x45,0x4e,0x43,0x4c, + 0x5f,0x50,0x4c,0x41,0x54,0x46,0x4f,0x52,0x4d,0x5f,0x4e,0x56,0x49,0x44,0x49,0x41,0x20,0x0a,0x5f,0x5f,0x61,0x74,0x74,0x72,0x69,0x62,0x75,0x74,0x65,0x5f,0x5f,0x28, + 0x28,0x72,0x65,0x71,0x64,0x5f,0x77,0x6f,0x72,0x6b,0x5f,0x67,0x72,0x6f,0x75,0x70,0x5f,0x73,0x69,0x7a,0x65,0x28,0x47,0x52,0x4f,0x55,0x50,0x5f,0x53,0x49,0x5a,0x45, + 0x2c,0x31,0x2c,0x31,0x29,0x29,0x29,0x0a,0x23,0x65,0x6e,0x64,0x69,0x66,0x0a,0x5f,0x5f,0x6b,0x65,0x72,0x6e,0x65,0x6c,0x20,0x76,0x6f,0x69,0x64,0x20,0x70,0x72,0x6f, + 0x67,0x70,0x6f,0x77,0x5f,0x73,0x65,0x61,0x72,0x63,0x68,0x28,0x5f,0x5f,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x20,0x64,0x61,0x67,0x5f,0x74,0x20,0x63,0x6f,0x6e,0x73,0x74, + 0x2a,0x20,0x67,0x5f,0x64,0x61,0x67,0x2c,0x5f,0x5f,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x20,0x75,0x69,0x6e,0x74,0x2a,0x20,0x6a,0x6f,0x62,0x5f,0x62,0x6c,0x6f,0x62,0x2c, + 0x75,0x6c,0x6f,0x6e,0x67,0x20,0x74,0x61,0x72,0x67,0x65,0x74,0x2c,0x75,0x69,0x6e,0x74,0x20,0x68,0x61,0x63,0x6b,0x5f,0x66,0x61,0x6c,0x73,0x65,0x2c,0x76,0x6f,0x6c, + 0x61,0x74,0x69,0x6c,0x65,0x20,0x5f,0x5f,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x20,0x75,0x69,0x6e,0x74,0x2a,0x20,0x72,0x65,0x73,0x75,0x6c,0x74,0x73,0x2c,0x76,0x6f,0x6c, + 0x61,0x74,0x69,0x6c,0x65,0x20,0x5f,0x5f,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x20,0x75,0x69,0x6e,0x74,0x2a,0x20,0x73,0x74,0x6f,0x70,0x29,0x0a,0x7b,0x0a,0x63,0x6f,0x6e, + 0x73,0x74,0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6c,0x69,0x64,0x3d,0x67,0x65,0x74,0x5f,0x6c,0x6f,0x63,0x61,0x6c,0x5f,0x69,0x64,0x28,0x30,0x29,0x3b, + 0x0a,0x63,0x6f,0x6e,0x73,0x74,0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x67,0x69,0x64,0x3d,0x67,0x65,0x74,0x5f,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x5f,0x69, + 0x64,0x28,0x30,0x29,0x3b,0x0a,0x69,0x66,0x28,0x73,0x74,0x6f,0x70,0x5b,0x30,0x5d,0x29,0x20,0x7b,0x0a,0x69,0x66,0x28,0x6c,0x69,0x64,0x3d,0x3d,0x30,0x29,0x20,0x7b, + 0x0a,0x61,0x74,0x6f,0x6d,0x69,0x63,0x5f,0x69,0x6e,0x63,0x28,0x73,0x74,0x6f,0x70,0x2b,0x31,0x29,0x3b,0x0a,0x7d,0x0a,0x72,0x65,0x74,0x75,0x72,0x6e,0x3b,0x0a,0x7d, + 0x0a,0x5f,0x5f,0x6c,0x6f,0x63,0x61,0x6c,0x20,0x73,0x68,0x75,0x66,0x66,0x6c,0x65,0x5f,0x74,0x20,0x73,0x68,0x61,0x72,0x65,0x5b,0x48,0x41,0x53,0x48,0x45,0x53,0x5f, + 0x50,0x45,0x52,0x5f,0x47,0x52,0x4f,0x55,0x50,0x5d,0x3b,0x0a,0x5f,0x5f,0x6c,0x6f,0x63,0x61,0x6c,0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x63,0x5f,0x64, + 0x61,0x67,0x5b,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x43,0x41,0x43,0x48,0x45,0x5f,0x57,0x4f,0x52,0x44,0x53,0x5d,0x3b,0x0a,0x63,0x6f,0x6e,0x73,0x74,0x20,0x75, + 0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x3d,0x6c,0x69,0x64,0x26,0x28,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e, + 0x45,0x53,0x2d,0x31,0x29,0x3b,0x0a,0x63,0x6f,0x6e,0x73,0x74,0x20,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x67,0x72,0x6f,0x75,0x70,0x5f,0x69,0x64,0x3d,0x6c, + 0x69,0x64,0x2f,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e,0x45,0x53,0x3b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20, + 0x77,0x6f,0x72,0x64,0x3d,0x6c,0x69,0x64,0x2a,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x44,0x41,0x47,0x5f,0x4c,0x4f,0x41,0x44,0x53,0x3b,0x20,0x77,0x6f,0x72,0x64, + 0x3c,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x43,0x41,0x43,0x48,0x45,0x5f,0x57,0x4f,0x52,0x44,0x53,0x3b,0x20,0x77,0x6f,0x72,0x64,0x2b,0x3d,0x47,0x52,0x4f,0x55, + 0x50,0x5f,0x53,0x49,0x5a,0x45,0x2a,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x44,0x41,0x47,0x5f,0x4c,0x4f,0x41,0x44,0x53,0x29,0x0a,0x7b,0x0a,0x64,0x61,0x67,0x5f, + 0x74,0x20,0x6c,0x6f,0x61,0x64,0x3d,0x67,0x5f,0x64,0x61,0x67,0x5b,0x77,0x6f,0x72,0x64,0x2f,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x44,0x41,0x47,0x5f,0x4c,0x4f, + 0x41,0x44,0x53,0x5d,0x3b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x44,0x41, + 0x47,0x5f,0x4c,0x4f,0x41,0x44,0x53,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x63,0x5f,0x64,0x61,0x67,0x5b,0x77,0x6f,0x72,0x64,0x2b,0x69,0x5d,0x3d,0x6c,0x6f,0x61,0x64, + 0x2e,0x73,0x5b,0x69,0x5d,0x3b,0x0a,0x7d,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x68,0x61,0x73,0x68,0x5f,0x73,0x65,0x65,0x64,0x5b,0x32,0x5d,0x3b,0x20, + 0x0a,0x68,0x61,0x73,0x68,0x33,0x32,0x5f,0x74,0x20,0x64,0x69,0x67,0x65,0x73,0x74,0x3b,0x20,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x73,0x74,0x61,0x74, + 0x65,0x32,0x5b,0x38,0x5d,0x3b,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x73,0x74,0x61,0x74,0x65,0x5b,0x32,0x35,0x5d,0x3b,0x20,0x0a,0x66,0x6f, + 0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x31,0x30,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x73,0x74,0x61,0x74,0x65,0x5b,0x69,0x5d,0x3d, + 0x6a,0x6f,0x62,0x5f,0x62,0x6c,0x6f,0x62,0x5b,0x69,0x5d,0x3b,0x0a,0x73,0x74,0x61,0x74,0x65,0x5b,0x38,0x5d,0x3d,0x67,0x69,0x64,0x3b,0x0a,0x66,0x6f,0x72,0x20,0x28, + 0x69,0x6e,0x74,0x20,0x69,0x3d,0x31,0x30,0x3b,0x20,0x69,0x3c,0x32,0x35,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x73,0x74,0x61,0x74,0x65,0x5b,0x69,0x5d,0x3d,0x72,0x61, + 0x76,0x65,0x6e,0x63,0x6f,0x69,0x6e,0x5f,0x72,0x6e,0x64,0x63,0x5b,0x69,0x2d,0x31,0x30,0x5d,0x3b,0x0a,0x6b,0x65,0x63,0x63,0x61,0x6b,0x5f,0x66,0x38,0x30,0x30,0x28, + 0x73,0x74,0x61,0x74,0x65,0x29,0x3b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x38,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a, + 0x73,0x74,0x61,0x74,0x65,0x32,0x5b,0x69,0x5d,0x3d,0x73,0x74,0x61,0x74,0x65,0x5b,0x69,0x5d,0x3b,0x0a,0x7d,0x0a,0x23,0x70,0x72,0x61,0x67,0x6d,0x61,0x20,0x75,0x6e, + 0x72,0x6f,0x6c,0x6c,0x20,0x31,0x0a,0x66,0x6f,0x72,0x20,0x28,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x68,0x3d,0x30,0x3b,0x20,0x68,0x3c,0x50,0x52,0x4f,0x47, + 0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e,0x45,0x53,0x3b,0x20,0x68,0x2b,0x2b,0x29,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6d,0x69,0x78,0x5b,0x50, + 0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x52,0x45,0x47,0x53,0x5d,0x3b,0x0a,0x69,0x66,0x28,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x3d,0x3d,0x68,0x29,0x20,0x7b,0x0a,0x73, + 0x68,0x61,0x72,0x65,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f,0x69,0x64,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x30,0x5d,0x3d,0x73,0x74,0x61,0x74,0x65,0x32, + 0x5b,0x30,0x5d,0x3b,0x0a,0x73,0x68,0x61,0x72,0x65,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f,0x69,0x64,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x31,0x5d,0x3d, + 0x73,0x74,0x61,0x74,0x65,0x32,0x5b,0x31,0x5d,0x3b,0x0a,0x7d,0x0a,0x62,0x61,0x72,0x72,0x69,0x65,0x72,0x28,0x43,0x4c,0x4b,0x5f,0x4c,0x4f,0x43,0x41,0x4c,0x5f,0x4d, + 0x45,0x4d,0x5f,0x46,0x45,0x4e,0x43,0x45,0x29,0x3b,0x0a,0x66,0x69,0x6c,0x6c,0x5f,0x6d,0x69,0x78,0x28,0x73,0x68,0x61,0x72,0x65,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f, + 0x69,0x64,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x2c,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x2c,0x6d,0x69,0x78,0x29,0x3b,0x0a,0x23,0x70,0x72,0x61,0x67,0x6d, + 0x61,0x20,0x75,0x6e,0x72,0x6f,0x6c,0x6c,0x20,0x32,0x0a,0x66,0x6f,0x72,0x20,0x28,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6c,0x6f,0x6f,0x70,0x3d,0x30,0x3b, + 0x20,0x6c,0x6f,0x6f,0x70,0x3c,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x43,0x4e,0x54,0x5f,0x44,0x41,0x47,0x3b,0x20,0x2b,0x2b,0x6c,0x6f,0x6f,0x70,0x29,0x0a,0x7b, + 0x0a,0x69,0x66,0x28,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x3d,0x3d,0x28,0x6c,0x6f,0x6f,0x70,0x20,0x25,0x20,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e, + 0x45,0x53,0x29,0x29,0x0a,0x73,0x68,0x61,0x72,0x65,0x5b,0x30,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f,0x69,0x64,0x5d,0x3d, + 0x6d,0x69,0x78,0x5b,0x30,0x5d,0x3b,0x0a,0x62,0x61,0x72,0x72,0x69,0x65,0x72,0x28,0x43,0x4c,0x4b,0x5f,0x4c,0x4f,0x43,0x41,0x4c,0x5f,0x4d,0x45,0x4d,0x5f,0x46,0x45, + 0x4e,0x43,0x45,0x29,0x3b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x6f,0x66,0x66,0x73,0x65,0x74,0x3d,0x73,0x68,0x61,0x72,0x65,0x5b,0x30,0x5d,0x2e,0x75, + 0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f,0x69,0x64,0x5d,0x3b,0x0a,0x6f,0x66,0x66,0x73,0x65,0x74,0x20,0x25,0x3d,0x20,0x50,0x52,0x4f,0x47, + 0x50,0x4f,0x57,0x5f,0x44,0x41,0x47,0x5f,0x45,0x4c,0x45,0x4d,0x45,0x4e,0x54,0x53,0x3b,0x0a,0x6f,0x66,0x66,0x73,0x65,0x74,0x3d,0x6f,0x66,0x66,0x73,0x65,0x74,0x2a, + 0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e,0x45,0x53,0x2b,0x28,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x5e,0x6c,0x6f,0x6f,0x70,0x29,0x20,0x25,0x20,0x50, + 0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e,0x45,0x53,0x3b,0x0a,0x64,0x61,0x67,0x5f,0x74,0x20,0x64,0x61,0x74,0x61,0x5f,0x64,0x61,0x67,0x3d,0x67,0x5f,0x64, + 0x61,0x67,0x5b,0x6f,0x66,0x66,0x73,0x65,0x74,0x5d,0x3b,0x0a,0x69,0x66,0x28,0x68,0x61,0x63,0x6b,0x5f,0x66,0x61,0x6c,0x73,0x65,0x29,0x20,0x62,0x61,0x72,0x72,0x69, + 0x65,0x72,0x28,0x43,0x4c,0x4b,0x5f,0x4c,0x4f,0x43,0x41,0x4c,0x5f,0x4d,0x45,0x4d,0x5f,0x46,0x45,0x4e,0x43,0x45,0x29,0x3b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f, + 0x74,0x20,0x64,0x61,0x74,0x61,0x3b,0x0a,0x58,0x4d,0x52,0x49,0x47,0x5f,0x49,0x4e,0x43,0x4c,0x55,0x44,0x45,0x5f,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x52,0x41, + 0x4e,0x44,0x4f,0x4d,0x5f,0x4d,0x41,0x54,0x48,0x0a,0x69,0x66,0x28,0x68,0x61,0x63,0x6b,0x5f,0x66,0x61,0x6c,0x73,0x65,0x29,0x20,0x62,0x61,0x72,0x72,0x69,0x65,0x72, + 0x28,0x43,0x4c,0x4b,0x5f,0x4c,0x4f,0x43,0x41,0x4c,0x5f,0x4d,0x45,0x4d,0x5f,0x46,0x45,0x4e,0x43,0x45,0x29,0x3b,0x0a,0x58,0x4d,0x52,0x49,0x47,0x5f,0x49,0x4e,0x43, + 0x4c,0x55,0x44,0x45,0x5f,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x44,0x41,0x54,0x41,0x5f,0x4c,0x4f,0x41,0x44,0x53,0x0a,0x7d,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32, + 0x5f,0x74,0x20,0x6d,0x69,0x78,0x5f,0x68,0x61,0x73,0x68,0x3d,0x46,0x4e,0x56,0x5f,0x4f,0x46,0x46,0x53,0x45,0x54,0x5f,0x42,0x41,0x53,0x49,0x53,0x3b,0x0a,0x23,0x70, + 0x72,0x61,0x67,0x6d,0x61,0x20,0x75,0x6e,0x72,0x6f,0x6c,0x6c,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x50,0x52,0x4f, + 0x47,0x50,0x4f,0x57,0x5f,0x52,0x45,0x47,0x53,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x66,0x6e,0x76,0x31,0x61,0x28,0x6d,0x69,0x78,0x5f,0x68,0x61,0x73,0x68,0x2c,0x6d, + 0x69,0x78,0x5b,0x69,0x5d,0x29,0x3b,0x0a,0x68,0x61,0x73,0x68,0x33,0x32,0x5f,0x74,0x20,0x64,0x69,0x67,0x65,0x73,0x74,0x5f,0x74,0x65,0x6d,0x70,0x3b,0x0a,0x66,0x6f, + 0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x38,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x64,0x69,0x67,0x65,0x73,0x74,0x5f,0x74,0x65,0x6d, + 0x70,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x69,0x5d,0x3d,0x46,0x4e,0x56,0x5f,0x4f,0x46,0x46,0x53,0x45,0x54,0x5f,0x42,0x41,0x53,0x49,0x53,0x3b,0x0a,0x73, + 0x68,0x61,0x72,0x65,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f,0x69,0x64,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x6c,0x61,0x6e,0x65,0x5f,0x69,0x64,0x5d,0x3d, + 0x6d,0x69,0x78,0x5f,0x68,0x61,0x73,0x68,0x3b,0x0a,0x62,0x61,0x72,0x72,0x69,0x65,0x72,0x28,0x43,0x4c,0x4b,0x5f,0x4c,0x4f,0x43,0x41,0x4c,0x5f,0x4d,0x45,0x4d,0x5f, + 0x46,0x45,0x4e,0x43,0x45,0x29,0x3b,0x0a,0x23,0x70,0x72,0x61,0x67,0x6d,0x61,0x20,0x75,0x6e,0x72,0x6f,0x6c,0x6c,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20, + 0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x50,0x52,0x4f,0x47,0x50,0x4f,0x57,0x5f,0x4c,0x41,0x4e,0x45,0x53,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x66,0x6e,0x76,0x31,0x61, + 0x28,0x64,0x69,0x67,0x65,0x73,0x74,0x5f,0x74,0x65,0x6d,0x70,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x69,0x20,0x25,0x20,0x38,0x5d,0x2c,0x73,0x68,0x61,0x72, + 0x65,0x5b,0x67,0x72,0x6f,0x75,0x70,0x5f,0x69,0x64,0x5d,0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x69,0x5d,0x29,0x3b,0x0a,0x69,0x66,0x28,0x68,0x3d,0x3d,0x6c, + 0x61,0x6e,0x65,0x5f,0x69,0x64,0x29,0x0a,0x64,0x69,0x67,0x65,0x73,0x74,0x3d,0x64,0x69,0x67,0x65,0x73,0x74,0x5f,0x74,0x65,0x6d,0x70,0x3b,0x0a,0x7d,0x0a,0x75,0x69, + 0x6e,0x74,0x36,0x34,0x5f,0x74,0x20,0x72,0x65,0x73,0x75,0x6c,0x74,0x3b,0x0a,0x7b,0x0a,0x75,0x69,0x6e,0x74,0x33,0x32,0x5f,0x74,0x20,0x73,0x74,0x61,0x74,0x65,0x5b, + 0x32,0x35,0x5d,0x3d,0x7b,0x30,0x78,0x30,0x7d,0x3b,0x20,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x30,0x3b,0x20,0x69,0x3c,0x38,0x3b,0x20,0x69, + 0x2b,0x2b,0x29,0x0a,0x73,0x74,0x61,0x74,0x65,0x5b,0x69,0x5d,0x3d,0x73,0x74,0x61,0x74,0x65,0x32,0x5b,0x69,0x5d,0x3b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74, + 0x20,0x69,0x3d,0x38,0x3b,0x20,0x69,0x3c,0x31,0x36,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x73,0x74,0x61,0x74,0x65,0x5b,0x69,0x5d,0x3d,0x64,0x69,0x67,0x65,0x73,0x74, + 0x2e,0x75,0x69,0x6e,0x74,0x33,0x32,0x73,0x5b,0x69,0x2d,0x38,0x5d,0x3b,0x0a,0x66,0x6f,0x72,0x20,0x28,0x69,0x6e,0x74,0x20,0x69,0x3d,0x31,0x36,0x3b,0x20,0x69,0x3c, + 0x32,0x35,0x3b,0x20,0x69,0x2b,0x2b,0x29,0x0a,0x73,0x74,0x61,0x74,0x65,0x5b,0x69,0x5d,0x3d,0x72,0x61,0x76,0x65,0x6e,0x63,0x6f,0x69,0x6e,0x5f,0x72,0x6e,0x64,0x63, + 0x5b,0x69,0x2d,0x31,0x36,0x5d,0x3b,0x0a,0x6b,0x65,0x63,0x63,0x61,0x6b,0x5f,0x66,0x38,0x30,0x30,0x28,0x73,0x74,0x61,0x74,0x65,0x29,0x3b,0x0a,0x75,0x69,0x6e,0x74, + 0x36,0x34,0x5f,0x74,0x20,0x72,0x65,0x73,0x3d,0x28,0x75,0x69,0x6e,0x74,0x36,0x34,0x5f,0x74,0x29,0x73,0x74,0x61,0x74,0x65,0x5b,0x31,0x5d,0x3c,0x3c,0x33,0x32,0x7c, + 0x73,0x74,0x61,0x74,0x65,0x5b,0x30,0x5d,0x3b,0x0a,0x72,0x65,0x73,0x75,0x6c,0x74,0x3d,0x61,0x73,0x5f,0x75,0x6c,0x6f,0x6e,0x67,0x28,0x61,0x73,0x5f,0x75,0x63,0x68, + 0x61,0x72,0x38,0x28,0x72,0x65,0x73,0x29,0x2e,0x73,0x37,0x36,0x35,0x34,0x33,0x32,0x31,0x30,0x29,0x3b,0x0a,0x7d,0x0a,0x69,0x66,0x28,0x72,0x65,0x73,0x75,0x6c,0x74, + 0x3c,0x3d,0x74,0x61,0x72,0x67,0x65,0x74,0x29,0x0a,0x7b,0x0a,0x2a,0x73,0x74,0x6f,0x70,0x3d,0x31,0x3b,0x0a,0x63,0x6f,0x6e,0x73,0x74,0x20,0x75,0x69,0x6e,0x74,0x20, + 0x6b,0x3d,0x61,0x74,0x6f,0x6d,0x69,0x63,0x5f,0x69,0x6e,0x63,0x28,0x72,0x65,0x73,0x75,0x6c,0x74,0x73,0x29,0x2b,0x31,0x3b,0x0a,0x69,0x66,0x28,0x6b,0x3c,0x3d,0x31, + 0x35,0x29,0x0a,0x72,0x65,0x73,0x75,0x6c,0x74,0x73,0x5b,0x6b,0x5d,0x3d,0x67,0x69,0x64,0x3b,0x0a,0x7d,0x0a,0x7d,0x0a,0x00 }; } // namespace xmrig diff --git a/src/backend/opencl/opencl.cmake b/src/backend/opencl/opencl.cmake index b5b8da25..6c6adc23 100644 --- a/src/backend/opencl/opencl.cmake +++ b/src/backend/opencl/opencl.cmake @@ -5,13 +5,7 @@ if (BUILD_STATIC AND XMRIG_OS_UNIX AND WITH_OPENCL) endif() if (WITH_OPENCL) - add_definitions(/DXMRIG_FEATURE_OPENCL) - add_definitions(/DCL_USE_DEPRECATED_OPENCL_1_2_APIS) - if (XMRIG_OS_APPLE) - add_definitions(/DCL_TARGET_OPENCL_VERSION=120) - elseif (WITH_OPENCL_VERSION) - add_definitions(/DCL_TARGET_OPENCL_VERSION=${WITH_OPENCL_VERSION}) - endif() + add_definitions(/DXMRIG_FEATURE_OPENCL /DCL_USE_DEPRECATED_OPENCL_1_2_APIS) set(HEADERS_BACKEND_OPENCL src/backend/opencl/cl/OclSource.h @@ -71,6 +65,13 @@ if (WITH_OPENCL) src/backend/opencl/wrappers/OclPlatform.cpp ) + if (XMRIG_OS_APPLE) + add_definitions(/DCL_TARGET_OPENCL_VERSION=120) + list(APPEND SOURCES_BACKEND_OPENCL src/backend/opencl/wrappers/OclDevice_mac.cpp) + elseif (WITH_OPENCL_VERSION) + add_definitions(/DCL_TARGET_OPENCL_VERSION=${WITH_OPENCL_VERSION}) + endif() + if (WIN32) list(APPEND SOURCES_BACKEND_OPENCL src/backend/opencl/OclCache_win.cpp) else() diff --git a/src/backend/opencl/wrappers/OclDevice.cpp b/src/backend/opencl/wrappers/OclDevice.cpp index bd1ccb45..846b5972 100644 --- a/src/backend/opencl/wrappers/OclDevice.cpp +++ b/src/backend/opencl/wrappers/OclDevice.cpp @@ -1,6 +1,7 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2021 Spudz76 + * Copyright (c) 2018-2024 SChernykh + * Copyright (c) 2016-2024 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -17,6 +18,7 @@ */ #include "backend/opencl/wrappers/OclDevice.h" +#include "3rdparty/fmt/core.h" #include "3rdparty/rapidjson/document.h" #include "backend/opencl/OclGenerator.h" #include "backend/opencl/OclThreads.h" @@ -30,19 +32,21 @@ #include - - -// NOLINTNEXTLINE(modernize-use-using) -typedef union -{ - struct { cl_uint type; cl_uint data[5]; } raw; - struct { cl_uint type; cl_char unused[17]; cl_char bus; cl_char device; cl_char function; } pcie; -} topology_amd; +#include namespace xmrig { +struct topology_amd { + cl_uint type; + cl_char unused[17]; + cl_char bus; + cl_char device; + cl_char function; +}; + + #ifdef XMRIG_ALGO_RANDOMX extern bool ocl_generic_rx_generator(const OclDevice &device, const Algorithm &algorithm, OclThreads &threads); #endif @@ -81,9 +85,11 @@ static OclVendor getPlatformVendorId(const String &vendor, const String &extensi return OCL_VENDOR_INTEL; } +# ifdef XMRIG_OS_APPLE if (extensions.contains("cl_APPLE_") || vendor.contains("Apple")) { return OCL_VENDOR_APPLE; } +# endif return OCL_VENDOR_UNKNOWN; } @@ -103,117 +109,16 @@ static OclVendor getVendorId(const String &vendor) return OCL_VENDOR_INTEL; } +# ifdef XMRIG_OS_APPLE if (vendor.contains("Apple")) { return OCL_VENDOR_APPLE; } +# endif return OCL_VENDOR_UNKNOWN; } -static OclDevice::Type getType(const String &name, const OclVendor platformVendorId) -{ - if (platformVendorId == OCL_VENDOR_APPLE) { - // Apple Platform: uses product names, not gfx# or codenames - if (name.contains("AMD Radeon")) { - if (name.contains(" 450 ") || - name.contains(" 455 ") || - name.contains(" 460 ")) { - return OclDevice::Baffin; - } - - if (name.contains(" 555 ") || name.contains(" 555X ") || - name.contains(" 560 ") || name.contains(" 560X ") || - name.contains(" 570 ") || name.contains(" 570X ") || - name.contains(" 575 ") || name.contains(" 575X ")) { - return OclDevice::Polaris; - } - - if (name.contains(" 580 ") || name.contains(" 580X ")) { - return OclDevice::Ellesmere; - } - - if (name.contains(" Vega ")) { - if (name.contains(" 48 ") || - name.contains(" 56 ") || - name.contains(" 64 ") || - name.contains(" 64X ")) { - return OclDevice::Vega_10; - } - if (name.contains(" 16 ") || - name.contains(" 20 ") || - name.contains(" II ")) { - return OclDevice::Vega_20; - } - } - - if (name.contains(" 5700 ") || name.contains(" W5700X ")) { - return OclDevice::Navi_10; - } - - if (name.contains(" 5600 ") || name.contains(" 5600M ")) { - return OclDevice::Navi_12; - } - - if (name.contains(" 5300 ") || name.contains(" 5300M ") || - name.contains(" 5500 ") || name.contains(" 5500M ")) { - return OclDevice::Navi_14; - } - - if (name.contains(" W6800 ") || name.contains(" W6900X ")) { - return OclDevice::Navi_21; - } - } - } - - if (name == "gfx900" || name == "gfx901") { - return OclDevice::Vega_10; - } - - if (name == "gfx902" || name == "gfx903") { - return OclDevice::Raven; - } - - if (name == "gfx906" || name == "gfx907") { - return OclDevice::Vega_20; - } - - if (name == "gfx1010") { - return OclDevice::Navi_10; - } - - if (name == "gfx1011") { - return OclDevice::Navi_12; - } - - if (name == "gfx1012") { - return OclDevice::Navi_14; - } - - if (name == "gfx1030") { - return OclDevice::Navi_21; - } - - if (name == "gfx804") { - return OclDevice::Lexa; - } - - if (name == "Baffin") { - return OclDevice::Baffin; - } - - if (name.contains("Ellesmere")) { - return OclDevice::Ellesmere; - } - - if (name == "gfx803" || name.contains("polaris")) { - return OclDevice::Polaris; - } - - return OclDevice::Unknown; -} - - } // namespace xmrig @@ -231,21 +136,21 @@ xmrig::OclDevice::OclDevice(uint32_t index, cl_device_id id, cl_platform_id plat { m_vendorId = getVendorId(m_vendor); m_platformVendorId = getPlatformVendorId(m_platformVendor, m_extensions); - m_type = getType(m_name, m_platformVendorId); + m_type = getType(m_name); if (m_extensions.contains("cl_amd_device_attribute_query")) { - topology_amd topology; - - if (OclLib::getDeviceInfo(id, CL_DEVICE_TOPOLOGY_AMD, sizeof(topology), &topology, nullptr) == CL_SUCCESS && topology.raw.type == CL_DEVICE_TOPOLOGY_TYPE_PCIE_AMD) { - m_topology = PciTopology(static_cast(topology.pcie.bus), static_cast(topology.pcie.device), static_cast(topology.pcie.function)); + topology_amd topology{}; + if (OclLib::getDeviceInfo(id, CL_DEVICE_TOPOLOGY_AMD, sizeof(topology), &topology) == CL_SUCCESS && topology.type == CL_DEVICE_TOPOLOGY_TYPE_PCIE_AMD) { + m_topology = { topology.bus, topology.device, topology.function }; } + m_board = OclLib::getString(id, CL_DEVICE_BOARD_NAME_AMD); } else if (m_extensions.contains("cl_nv_device_attribute_query")) { cl_uint bus = 0; - if (OclLib::getDeviceInfo(id, CL_DEVICE_PCI_BUS_ID_NV, sizeof (bus), &bus, nullptr) == CL_SUCCESS) { + if (OclLib::getDeviceInfo(id, CL_DEVICE_PCI_BUS_ID_NV, sizeof(bus), &bus) == CL_SUCCESS) { cl_uint slot = OclLib::getUint(id, CL_DEVICE_PCI_SLOT_ID_NV); - m_topology = PciTopology(bus, (slot >> 3) & 0xff, slot & 7); + m_topology = { bus, (slot >> 3) & 0xff, slot & 7 }; } } } @@ -253,17 +158,11 @@ xmrig::OclDevice::OclDevice(uint32_t index, cl_device_id id, cl_platform_id plat xmrig::String xmrig::OclDevice::printableName() const { - const size_t size = m_board.size() + m_name.size() + 64; - char *buf = new char[size](); - if (m_board.isNull()) { - snprintf(buf, size, GREEN_BOLD("%s"), m_name.data()); - } - else { - snprintf(buf, size, GREEN_BOLD("%s") " (" CYAN_BOLD("%s") ")", m_board.data(), m_name.data()); + return fmt::format(GREEN_BOLD("{}"), m_name).c_str(); } - return buf; + return fmt::format(GREEN_BOLD("{}") " (" CYAN_BOLD("{}") ")", m_board, m_name).c_str(); } @@ -311,3 +210,35 @@ void xmrig::OclDevice::toJSON(rapidjson::Value &out, rapidjson::Document &doc) c # endif } #endif + + +#ifndef XMRIG_OS_APPLE +xmrig::OclDevice::Type xmrig::OclDevice::getType(const String &name) +{ + static std::map types = { + { "gfx900", Vega_10 }, + { "gfx901", Vega_10 }, + { "gfx902", Raven }, + { "gfx903", Raven }, + { "gfx906", Vega_20 }, + { "gfx907", Vega_20 }, + { "gfx1010", Navi_10 }, + { "gfx1011", Navi_12 }, + { "gfx1012", Navi_14 }, + { "gfx1030", Navi_21 }, + { "gfx804", Lexa }, + { "Baffin", Baffin }, + { "Ellesmere", Ellesmere }, + { "gfx803", Polaris }, + { "polaris", Polaris }, + }; + + for (auto &kv : types) { + if (name.contains(kv.first)) { + return kv.second; + } + } + + return OclDevice::Unknown; +} +#endif diff --git a/src/backend/opencl/wrappers/OclDevice.h b/src/backend/opencl/wrappers/OclDevice.h index 7e9e4155..356335fd 100644 --- a/src/backend/opencl/wrappers/OclDevice.h +++ b/src/backend/opencl/wrappers/OclDevice.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2024 SChernykh + * Copyright (c) 2016-2024 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -86,6 +86,8 @@ public: # endif private: + static OclDevice::Type getType(const String &name); + cl_device_id m_id = nullptr; cl_platform_id m_platform = nullptr; const String m_platformVendor; diff --git a/src/backend/opencl/wrappers/OclDevice_mac.cpp b/src/backend/opencl/wrappers/OclDevice_mac.cpp new file mode 100644 index 00000000..57b44056 --- /dev/null +++ b/src/backend/opencl/wrappers/OclDevice_mac.cpp @@ -0,0 +1,77 @@ +/* XMRig + * Copyright (c) 2021 Spudz76 + * Copyright (c) 2018-2024 SChernykh + * Copyright (c) 2016-2024 XMRig , + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "backend/opencl/wrappers/OclDevice.h" + + +xmrig::OclDevice::Type xmrig::OclDevice::getType(const String &name) +{ + // Apple Platform: uses product names, not gfx# or codenames + if (name.contains("AMD Radeon")) { + if (name.contains(" 450 ") || + name.contains(" 455 ") || + name.contains(" 460 ")) { + return Baffin; + } + + if (name.contains(" 555 ") || name.contains(" 555X ") || + name.contains(" 560 ") || name.contains(" 560X ") || + name.contains(" 570 ") || name.contains(" 570X ") || + name.contains(" 575 ") || name.contains(" 575X ")) { + return Polaris; + } + + if (name.contains(" 580 ") || name.contains(" 580X ")) { + return Ellesmere; + } + + if (name.contains(" Vega ")) { + if (name.contains(" 48 ") || + name.contains(" 56 ") || + name.contains(" 64 ") || + name.contains(" 64X ")) { + return Vega_10; + } + if (name.contains(" 16 ") || + name.contains(" 20 ") || + name.contains(" II ")) { + return Vega_20; + } + } + + if (name.contains(" 5700 ") || name.contains(" W5700X ")) { + return Navi_10; + } + + if (name.contains(" 5600 ") || name.contains(" 5600M ")) { + return Navi_12; + } + + if (name.contains(" 5300 ") || name.contains(" 5300M ") || + name.contains(" 5500 ") || name.contains(" 5500M ")) { + return Navi_14; + } + + if (name.contains(" W6800 ") || name.contains(" W6900X ")) { + return Navi_21; + } + } + + return OclDevice::Unknown; +} diff --git a/src/base/api/Api.cpp b/src/base/api/Api.cpp index 77f8e4aa..2556a502 100644 --- a/src/base/api/Api.cpp +++ b/src/base/api/Api.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2024 SChernykh - * Copyright (c) 2016-2024 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,6 +20,7 @@ #include "base/api/Api.h" +#include "3rdparty/rapidjson/writer.h" #include "base/api/interfaces/IApiListener.h" #include "base/api/requests/HttpApiRequest.h" #include "base/crypto/keccak.h" @@ -31,7 +32,6 @@ #include "base/tools/Chrono.h" #include "base/tools/Cvt.h" #include "core/config/Config.h" -#include "core/Controller.h" #include "version.h" @@ -46,6 +46,12 @@ namespace xmrig { +static_assert( + RAPIDJSON_WRITE_DEFAULT_FLAGS == (rapidjson::kWriteNanAndInfFlag | rapidjson::kWriteNanAndInfNullFlag), + "(rapidjson::kWriteNanAndInfFlag | rapidjson::kWriteNanAndInfNullFlag) required" + ); + + static rapidjson::Value getResources(rapidjson::Document &doc) { using namespace rapidjson; diff --git a/src/base/io/json/Json_win.cpp b/src/base/io/json/Json_win.cpp index 499ae129..9de1caf6 100644 --- a/src/base/io/json/Json_win.cpp +++ b/src/base/io/json/Json_win.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ #include -#ifdef __GNUC__ +#if defined(__GNUC__) && !defined(__clang__) # include # include # include @@ -39,7 +39,7 @@ namespace xmrig { -#if defined(_MSC_VER) || defined (__GNUC__) +#if defined(_MSC_VER) || defined(__GNUC__) static std::wstring toUtf16(const char *str) { const int size = static_cast(strlen(str)); @@ -56,9 +56,9 @@ static std::wstring toUtf16(const char *str) #endif -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(_LIBCPP_HAS_OPEN_WITH_WCHAR) # define OPEN_IFS(name) \ - std::ifstream ifs(toUtf16(name), std::ios_base::in | std::ios_base::binary); \ + std::ifstream ifs(toUtf16(name).c_str(), std::ios_base::in | std::ios_base::binary);\ if (!ifs.is_open()) { \ return false; \ } @@ -98,8 +98,8 @@ bool xmrig::Json::save(const char *fileName, const rapidjson::Document &doc) using namespace rapidjson; constexpr const std::ios_base::openmode mode = std::ios_base::out | std::ios_base::binary | std::ios_base::trunc; -# if defined(_MSC_VER) - std::ofstream ofs(toUtf16(fileName), mode); +# if defined(_MSC_VER) || defined(_LIBCPP_HAS_OPEN_WITH_WCHAR) + std::ofstream ofs(toUtf16(fileName).c_str(), mode); if (!ofs.is_open()) { return false; } diff --git a/src/base/kernel/Entry.cpp b/src/base/kernel/Entry.cpp index f12e833f..b99621b6 100644 --- a/src/base/kernel/Entry.cpp +++ b/src/base/kernel/Entry.cpp @@ -5,8 +5,8 @@ * Copyright 2014-2016 Wolf9466 * Copyright 2016 Jay D Dee * Copyright 2017-2018 XMR-Stak , - * Copyright 2018-2019 SChernykh - * Copyright 2016-2019 XMRig , + * Copyright 2018-2024 SChernykh + * Copyright 2016-2024 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -22,11 +22,9 @@ * along with this program. If not, see . */ - #include #include - #ifdef XMRIG_FEATURE_TLS # include #endif @@ -66,13 +64,13 @@ static int showVersion() # endif printf("\n features:" -# if defined(__i386__) || defined(_M_IX86) - " 32-bit" -# elif defined(__x86_64__) || defined(_M_AMD64) +# if defined(__x86_64__) || defined(_M_AMD64) || defined (__arm64__) || defined (__aarch64__) " 64-bit" +# else + " 32-bit" # endif -# if defined(__AES__) || defined(_MSC_VER) +# if defined(__AES__) || defined(_MSC_VER) || defined(__ARM_FEATURE_CRYPTO) " AES" # endif "\n"); diff --git a/src/base/kernel/Platform_win.cpp b/src/base/kernel/Platform_win.cpp index 94d1282e..b5973f3d 100644 --- a/src/base/kernel/Platform_win.cpp +++ b/src/base/kernel/Platform_win.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2023 SChernykh - * Copyright (c) 2016-2023 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -53,13 +53,21 @@ char *xmrig::Platform::createUserAgent() char *buf = new char[max](); int length = snprintf(buf, max, "%s/%s (Windows NT %lu.%lu", APP_NAME, APP_VERSION, osver.dwMajorVersion, osver.dwMinorVersion); -# if defined(__x86_64__) || defined(_M_AMD64) - length += snprintf(buf + length, max - length, "; Win64; x64) libuv/%s", uv_version_string()); +# if defined(XMRIG_64_BIT) + length += snprintf(buf + length, max - length, "; Win64; " +# if defined(XMRIG_ARM) + "arm64" +# else + "x64" +# endif + ") libuv/%s", uv_version_string()); # else length += snprintf(buf + length, max - length, ") libuv/%s", uv_version_string()); # endif -# ifdef __GNUC__ +# ifdef __clang__ + snprintf(buf + length, max - length, " clang/%d.%d.%d", __clang_major__, __clang_minor__, __clang_patchlevel__); +# elif defined(__GNUC__) snprintf(buf + length, max - length, " gcc/%d.%d.%d", __GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__); # elif _MSC_VER snprintf(buf + length, max - length, " msvc/%d", MSVC_VERSION); diff --git a/src/base/kernel/config/BaseTransform.cpp b/src/base/kernel/config/BaseTransform.cpp index a1b430da..339e4621 100644 --- a/src/base/kernel/config/BaseTransform.cpp +++ b/src/base/kernel/config/BaseTransform.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,14 +18,12 @@ #include - #ifdef _MSC_VER # include "getopt/getopt.h" #else # include #endif - #include "base/kernel/config/BaseTransform.h" #include "base/io/json/JsonChain.h" #include "base/io/log/Log.h" @@ -37,7 +35,6 @@ #include "base/net/stratum/Pools.h" #include "core/config/Config_platform.h" - #ifdef XMRIG_FEATURE_TLS # include "base/net/tls/TlsConfig.h" #endif @@ -47,9 +44,9 @@ void xmrig::BaseTransform::load(JsonChain &chain, Process *process, IConfigTrans { using namespace rapidjson; - int key = 0; - int argc = process->arguments().argc(); - char **argv = process->arguments().argv(); + int key = 0; + const int argc = process->arguments().argc(); + char **argv = process->arguments().argv(); Document doc(kObjectType); @@ -262,7 +259,8 @@ void xmrig::BaseTransform::transform(rapidjson::Document &doc, int key, const ch case IConfig::DaemonKey: /* --daemon */ case IConfig::SubmitToOriginKey: /* --submit-to-origin */ case IConfig::VerboseKey: /* --verbose */ - case IConfig::DnsIPv6Key: /* --dns-ipv6 */ + case IConfig::DnsIPv4Key: /* --ipv4 */ + case IConfig::DnsIPv6Key: /* --ipv6 */ return transformBoolean(doc, key, true); case IConfig::ColorKey: /* --no-color */ @@ -323,8 +321,11 @@ void xmrig::BaseTransform::transformBoolean(rapidjson::Document &doc, int key, b case IConfig::NoTitleKey: /* --no-title */ return set(doc, BaseConfig::kTitle, enable); - case IConfig::DnsIPv6Key: /* --dns-ipv6 */ - return set(doc, DnsConfig::kField, DnsConfig::kIPv6, enable); + case IConfig::DnsIPv4Key: /* --ipv4 */ + return set(doc, DnsConfig::kField, DnsConfig::kIPv, 4); + + case IConfig::DnsIPv6Key: /* --ipv6 */ + return set(doc, DnsConfig::kField, DnsConfig::kIPv, 6); default: break; diff --git a/src/base/kernel/interfaces/IConfig.h b/src/base/kernel/interfaces/IConfig.h index 98957fc4..94c5760d 100644 --- a/src/base/kernel/interfaces/IConfig.h +++ b/src/base/kernel/interfaces/IConfig.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,9 +16,7 @@ * along with this program. If not, see . */ -#ifndef XMRIG_ICONFIG_H -#define XMRIG_ICONFIG_H - +#pragma once #include "3rdparty/rapidjson/fwd.h" @@ -82,7 +80,8 @@ public: HugePageSizeKey = 1050, PauseOnActiveKey = 1051, SubmitToOriginKey = 1052, - DnsIPv6Key = 1053, + DnsIPv4Key = '4', + DnsIPv6Key = '6', DnsTtlKey = 1054, SpendSecretKey = 1055, DaemonZMQPortKey = 1056, @@ -177,7 +176,4 @@ public: }; -} /* namespace xmrig */ - - -#endif // XMRIG_ICONFIG_H +} // namespace xmrig diff --git a/src/base/kernel/interfaces/IDnsBackend.h b/src/base/kernel/interfaces/IDnsBackend.h index ca676f32..de069dc5 100644 --- a/src/base/kernel/interfaces/IDnsBackend.h +++ b/src/base/kernel/interfaces/IDnsBackend.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,21 +16,16 @@ * along with this program. If not, see . */ -#ifndef XMRIG_IDNSBACKEND_H -#define XMRIG_IDNSBACKEND_H - +#pragma once #include "base/tools/Object.h" -#include - - namespace xmrig { +class DnsConfig; class DnsRecords; -class DnsRequest; class IDnsListener; class String; @@ -43,12 +38,8 @@ public: IDnsBackend() = default; virtual ~IDnsBackend() = default; - virtual const DnsRecords &records() const = 0; - virtual std::shared_ptr resolve(const String &host, IDnsListener *listener, uint64_t ttl) = 0; + virtual void resolve(const String &host, const std::weak_ptr &listener, const DnsConfig &config) = 0; }; -} /* namespace xmrig */ - - -#endif // XMRIG_IDNSBACKEND_H +} // namespace xmrig diff --git a/src/base/net/dns/Dns.cpp b/src/base/net/dns/Dns.cpp index 85590bb7..d5e71b49 100644 --- a/src/base/net/dns/Dns.cpp +++ b/src/base/net/dns/Dns.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ #include "base/net/dns/Dns.h" +#include "base/net/dns/DnsRequest.h" #include "base/net/dns/DnsUvBackend.h" @@ -25,17 +26,21 @@ namespace xmrig { DnsConfig Dns::m_config; -std::map > Dns::m_backends; +std::map> Dns::m_backends; } // namespace xmrig -std::shared_ptr xmrig::Dns::resolve(const String &host, IDnsListener *listener, uint64_t ttl) +std::shared_ptr xmrig::Dns::resolve(const String &host, IDnsListener *listener) { + auto req = std::make_shared(listener); + if (m_backends.find(host) == m_backends.end()) { m_backends.insert({ host, std::make_shared() }); } - return m_backends.at(host)->resolve(host, listener, ttl == 0 ? m_config.ttl() : ttl); + m_backends.at(host)->resolve(host, req, m_config); + + return req; } diff --git a/src/base/net/dns/Dns.h b/src/base/net/dns/Dns.h index cf054390..2408245f 100644 --- a/src/base/net/dns/Dns.h +++ b/src/base/net/dns/Dns.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -43,7 +43,7 @@ public: inline static const DnsConfig &config() { return m_config; } inline static void set(const DnsConfig &config) { m_config = config; } - static std::shared_ptr resolve(const String &host, IDnsListener *listener, uint64_t ttl = 0); + static std::shared_ptr resolve(const String &host, IDnsListener *listener); private: static DnsConfig m_config; diff --git a/src/base/net/dns/DnsConfig.cpp b/src/base/net/dns/DnsConfig.cpp index f9ec7e28..3b2bbe73 100644 --- a/src/base/net/dns/DnsConfig.cpp +++ b/src/base/net/dns/DnsConfig.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,15 +20,15 @@ #include "3rdparty/rapidjson/document.h" #include "base/io/json/Json.h" - #include +#include namespace xmrig { const char *DnsConfig::kField = "dns"; -const char *DnsConfig::kIPv6 = "ipv6"; +const char *DnsConfig::kIPv = "ip_version"; const char *DnsConfig::kTTL = "ttl"; @@ -37,8 +37,26 @@ const char *DnsConfig::kTTL = "ttl"; xmrig::DnsConfig::DnsConfig(const rapidjson::Value &value) { - m_ipv6 = Json::getBool(value, kIPv6, m_ipv6); - m_ttl = std::max(Json::getUint(value, kTTL, m_ttl), 1U); + const uint32_t ipv = Json::getUint(value, kIPv, m_ipv); + if (ipv == 0 || ipv == 4 || ipv == 6) { + m_ipv = ipv; + } + + m_ttl = std::max(Json::getUint(value, kTTL, m_ttl), 1U); +} + + +int xmrig::DnsConfig::ai_family() const +{ + if (m_ipv == 4) { + return AF_INET; + } + + if (m_ipv == 6) { + return AF_INET6; + } + + return AF_UNSPEC; } @@ -49,8 +67,8 @@ rapidjson::Value xmrig::DnsConfig::toJSON(rapidjson::Document &doc) const auto &allocator = doc.GetAllocator(); Value obj(kObjectType); - obj.AddMember(StringRef(kIPv6), m_ipv6, allocator); - obj.AddMember(StringRef(kTTL), m_ttl, allocator); + obj.AddMember(StringRef(kIPv), m_ipv, allocator); + obj.AddMember(StringRef(kTTL), m_ttl, allocator); return obj; } diff --git a/src/base/net/dns/DnsConfig.h b/src/base/net/dns/DnsConfig.h index 605e837b..7c985f90 100644 --- a/src/base/net/dns/DnsConfig.h +++ b/src/base/net/dns/DnsConfig.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,9 +16,7 @@ * along with this program. If not, see . */ -#ifndef XMRIG_DNSCONFIG_H -#define XMRIG_DNSCONFIG_H - +#pragma once #include "3rdparty/rapidjson/fwd.h" @@ -30,25 +28,22 @@ class DnsConfig { public: static const char *kField; - static const char *kIPv6; + static const char *kIPv; static const char *kTTL; DnsConfig() = default; DnsConfig(const rapidjson::Value &value); - inline bool isIPv6() const { return m_ipv6; } + inline uint32_t ipv() const { return m_ipv; } inline uint32_t ttl() const { return m_ttl * 1000U; } + int ai_family() const; rapidjson::Value toJSON(rapidjson::Document &doc) const; - private: - bool m_ipv6 = false; - uint32_t m_ttl = 30U; + uint32_t m_ttl = 30U; + uint32_t m_ipv = 0U; }; -} /* namespace xmrig */ - - -#endif /* XMRIG_DNSCONFIG_H */ +} // namespace xmrig diff --git a/src/base/net/dns/DnsRecord.cpp b/src/base/net/dns/DnsRecord.cpp index 3bf8f097..2b5a5058 100644 --- a/src/base/net/dns/DnsRecord.cpp +++ b/src/base/net/dns/DnsRecord.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2023 SChernykh - * Copyright (c) 2016-2023 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,19 +16,16 @@ * along with this program. If not, see . */ - #include - #include "base/net/dns/DnsRecord.h" -xmrig::DnsRecord::DnsRecord(const addrinfo *addr) : - m_type(addr->ai_family == AF_INET6 ? AAAA : (addr->ai_family == AF_INET ? A : Unknown)) +xmrig::DnsRecord::DnsRecord(const addrinfo *addr) { static_assert(sizeof(m_data) >= sizeof(sockaddr_in6), "Not enough storage for IPv6 address."); - memcpy(m_data, addr->ai_addr, m_type == AAAA ? sizeof(sockaddr_in6) : sizeof(sockaddr_in)); + memcpy(m_data, addr->ai_addr, addr->ai_family == AF_INET6 ? sizeof(sockaddr_in6) : sizeof(sockaddr_in)); } @@ -44,7 +41,7 @@ xmrig::String xmrig::DnsRecord::ip() const { char *buf = nullptr; - if (m_type == AAAA) { + if (reinterpret_cast(m_data).sa_family == AF_INET6) { buf = new char[45](); uv_ip6_name(reinterpret_cast(m_data), buf, 45); } diff --git a/src/base/net/dns/DnsRecord.h b/src/base/net/dns/DnsRecord.h index 7a68ea2f..548b603f 100644 --- a/src/base/net/dns/DnsRecord.h +++ b/src/base/net/dns/DnsRecord.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,14 +16,11 @@ * along with this program. If not, see . */ -#ifndef XMRIG_DNSRECORD_H -#define XMRIG_DNSRECORD_H - +#pragma once struct addrinfo; struct sockaddr; - #include "base/tools/String.h" @@ -33,28 +30,15 @@ namespace xmrig { class DnsRecord { public: - enum Type : uint32_t { - Unknown, - A, - AAAA - }; - DnsRecord() {} DnsRecord(const addrinfo *addr); const sockaddr *addr(uint16_t port = 0) const; String ip() const; - inline bool isValid() const { return m_type != Unknown; } - inline Type type() const { return m_type; } - private: mutable uint8_t m_data[28]{}; - const Type m_type = Unknown; }; -} /* namespace xmrig */ - - -#endif /* XMRIG_DNSRECORD_H */ +} // namespace xmrig diff --git a/src/base/net/dns/DnsRecords.cpp b/src/base/net/dns/DnsRecords.cpp index 072f9672..aff2b323 100644 --- a/src/base/net/dns/DnsRecords.cpp +++ b/src/base/net/dns/DnsRecords.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,90 +18,96 @@ #include - #include "base/net/dns/DnsRecords.h" -#include "base/net/dns/Dns.h" -const xmrig::DnsRecord &xmrig::DnsRecords::get(DnsRecord::Type prefered) const +namespace { + + +static size_t dns_records_count(const addrinfo *res, int &ai_family) +{ + size_t ipv4 = 0; + size_t ipv6 = 0; + + while (res != nullptr) { + if (res->ai_family == AF_INET) { + ++ipv4; + } + + if (res->ai_family == AF_INET6) { + ++ipv6; + } + + res = res->ai_next; + } + + if (ai_family == AF_INET6 && !ipv6) { + ai_family = AF_INET; + } + + switch (ai_family) { + case AF_UNSPEC: + return ipv4 + ipv6; + + case AF_INET: + return ipv4; + + case AF_INET6: + return ipv6; + + default: + break; + } + + return 0; +} + + +} // namespace + + +xmrig::DnsRecords::DnsRecords(const addrinfo *res, int ai_family) +{ + size_t size = dns_records_count(res, ai_family); + if (!size) { + return; + } + + m_records.reserve(size); + + if (ai_family == AF_UNSPEC) { + while (res != nullptr) { + if (res->ai_family == AF_INET || res->ai_family == AF_INET6) { + m_records.emplace_back(res); + } + + res = res->ai_next; + }; + } else { + while (res != nullptr) { + if (res->ai_family == ai_family) { + m_records.emplace_back(res); + } + + res = res->ai_next; + }; + } + + size = m_records.size(); + if (size > 1) { + m_index = static_cast(rand()) % size; // NOLINT(concurrency-mt-unsafe, cert-msc30-c, cert-msc50-cpp) + } +} + + +const xmrig::DnsRecord &xmrig::DnsRecords::get() const { static const DnsRecord defaultRecord; - if (isEmpty()) { - return defaultRecord; - } - - const size_t ipv4 = m_ipv4.size(); - const size_t ipv6 = m_ipv6.size(); - - if (ipv6 && (prefered == DnsRecord::AAAA || Dns::config().isIPv6() || !ipv4)) { - return m_ipv6[ipv6 == 1 ? 0 : static_cast(rand()) % ipv6]; // NOLINT(concurrency-mt-unsafe, cert-msc30-c, cert-msc50-cpp) - } - - if (ipv4) { - return m_ipv4[ipv4 == 1 ? 0 : static_cast(rand()) % ipv4]; // NOLINT(concurrency-mt-unsafe, cert-msc30-c, cert-msc50-cpp) + const size_t size = m_records.size(); + if (size > 0) { + return m_records[m_index++ % size]; } return defaultRecord; } - - -size_t xmrig::DnsRecords::count(DnsRecord::Type type) const -{ - if (type == DnsRecord::A) { - return m_ipv4.size(); - } - - if (type == DnsRecord::AAAA) { - return m_ipv6.size(); - } - - return m_ipv4.size() + m_ipv6.size(); -} - - -void xmrig::DnsRecords::clear() -{ - m_ipv4.clear(); - m_ipv6.clear(); -} - - -void xmrig::DnsRecords::parse(addrinfo *res) -{ - clear(); - - addrinfo *ptr = res; - size_t ipv4 = 0; - size_t ipv6 = 0; - - while (ptr != nullptr) { - if (ptr->ai_family == AF_INET) { - ++ipv4; - } - else if (ptr->ai_family == AF_INET6) { - ++ipv6; - } - - ptr = ptr->ai_next; - } - - if (ipv4 == 0 && ipv6 == 0) { - return; - } - - m_ipv4.reserve(ipv4); - m_ipv6.reserve(ipv6); - - ptr = res; - while (ptr != nullptr) { - if (ptr->ai_family == AF_INET) { - m_ipv4.emplace_back(ptr); - } - else if (ptr->ai_family == AF_INET6) { - m_ipv6.emplace_back(ptr); - } - - ptr = ptr->ai_next; - } -} diff --git a/src/base/net/dns/DnsRecords.h b/src/base/net/dns/DnsRecords.h index cfa19217..4b16d54a 100644 --- a/src/base/net/dns/DnsRecords.h +++ b/src/base/net/dns/DnsRecords.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,9 +16,7 @@ * along with this program. If not, see . */ -#ifndef XMRIG_DNSRECORDS_H -#define XMRIG_DNSRECORDS_H - +#pragma once #include "base/net/dns/DnsRecord.h" @@ -29,20 +27,19 @@ namespace xmrig { class DnsRecords { public: - inline bool isEmpty() const { return m_ipv4.empty() && m_ipv6.empty(); } + DnsRecords() = default; + DnsRecords(const addrinfo *res, int ai_family); - const DnsRecord &get(DnsRecord::Type prefered = DnsRecord::Unknown) const; - size_t count(DnsRecord::Type type = DnsRecord::Unknown) const; - void clear(); - void parse(addrinfo *res); + inline bool isEmpty() const { return m_records.empty(); } + inline const std::vector &records() const { return m_records; } + inline size_t size() const { return m_records.size(); } + + const DnsRecord &get() const; private: - std::vector m_ipv4; - std::vector m_ipv6; + mutable size_t m_index = 0; + std::vector m_records; }; -} /* namespace xmrig */ - - -#endif /* XMRIG_DNSRECORDS_H */ +} // namespace xmrig diff --git a/src/base/net/dns/DnsRequest.h b/src/base/net/dns/DnsRequest.h index 036eaa34..175b9c07 100644 --- a/src/base/net/dns/DnsRequest.h +++ b/src/base/net/dns/DnsRequest.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,35 +16,30 @@ * along with this program. If not, see . */ -#ifndef XMRIG_DNSREQUEST_H -#define XMRIG_DNSREQUEST_H +#pragma once - -#include "base/tools/Object.h" - - -#include +#include "base/kernel/interfaces/IDnsListener.h" namespace xmrig { -class IDnsListener; - - -class DnsRequest +class DnsRequest : public IDnsListener { public: XMRIG_DISABLE_COPY_MOVE_DEFAULT(DnsRequest) - DnsRequest(IDnsListener *listener) : listener(listener) {} - ~DnsRequest() = default; + inline DnsRequest(IDnsListener *listener) : m_listener(listener) {} + ~DnsRequest() override = default; - IDnsListener *listener; +protected: + inline void onResolved(const DnsRecords &records, int status, const char *error) override { + m_listener->onResolved(records, status, error); + } + +private: + IDnsListener *m_listener; }; -} /* namespace xmrig */ - - -#endif /* XMRIG_DNSREQUEST_H */ +} // namespace xmrig diff --git a/src/base/net/dns/DnsUvBackend.cpp b/src/base/net/dns/DnsUvBackend.cpp index 33f27f95..850f493e 100644 --- a/src/base/net/dns/DnsUvBackend.cpp +++ b/src/base/net/dns/DnsUvBackend.cpp @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2023 SChernykh - * Copyright (c) 2016-2023 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,13 +16,11 @@ * along with this program. If not, see . */ - #include - #include "base/net/dns/DnsUvBackend.h" #include "base/kernel/interfaces/IDnsListener.h" -#include "base/net/dns/DnsRequest.h" +#include "base/net/dns/DnsConfig.h" #include "base/tools/Chrono.h" @@ -73,21 +71,23 @@ xmrig::DnsUvBackend::~DnsUvBackend() } -std::shared_ptr xmrig::DnsUvBackend::resolve(const String &host, IDnsListener *listener, uint64_t ttl) +void xmrig::DnsUvBackend::resolve(const String &host, const std::weak_ptr &listener, const DnsConfig &config) { - auto req = std::make_shared(listener); + m_queue.emplace_back(listener); - if (Chrono::currentMSecsSinceEpoch() - m_ts <= ttl && !m_records.isEmpty()) { - req->listener->onResolved(m_records, 0, nullptr); - } else { - m_queue.emplace(req); + if (Chrono::currentMSecsSinceEpoch() - m_ts <= config.ttl()) { + return notify(); } - if (m_queue.size() == 1 && !resolve(host)) { - done(); + if (m_req) { + return; } - return req; + m_ai_family = config.ai_family(); + + if (!resolve(host)) { + notify(); + } } @@ -102,44 +102,46 @@ bool xmrig::DnsUvBackend::resolve(const String &host) } -void xmrig::DnsUvBackend::done() +void xmrig::DnsUvBackend::notify() { const char *error = m_status < 0 ? uv_strerror(m_status) : nullptr; - while (!m_queue.empty()) { - auto req = std::move(m_queue.front()).lock(); - if (req) { - req->listener->onResolved(m_records, m_status, error); + for (const auto &l : m_queue) { + auto listener = l.lock(); + if (listener) { + listener->onResolved(m_records, m_status, error); } - - m_queue.pop(); } + m_queue.clear(); m_req.reset(); } void xmrig::DnsUvBackend::onResolved(int status, addrinfo *res) { - m_ts = Chrono::currentMSecsSinceEpoch(); + m_status = status; + m_ts = Chrono::currentMSecsSinceEpoch(); - if ((m_status = status) < 0) { - return done(); + if (m_status < 0) { + m_records = {}; + + return notify(); } - m_records.parse(res); + m_records = { res, m_ai_family }; if (m_records.isEmpty()) { m_status = UV_EAI_NONAME; } - done(); + notify(); } void xmrig::DnsUvBackend::onResolved(uv_getaddrinfo_t *req, int status, addrinfo *res) { - auto backend = getStorage().get(req->data); + auto *backend = getStorage().get(req->data); if (backend) { backend->onResolved(status, res); } diff --git a/src/base/net/dns/DnsUvBackend.h b/src/base/net/dns/DnsUvBackend.h index 3c2436c7..24a6e1aa 100644 --- a/src/base/net/dns/DnsUvBackend.h +++ b/src/base/net/dns/DnsUvBackend.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,16 +16,13 @@ * along with this program. If not, see . */ -#ifndef XMRIG_DNSUVBACKEND_H -#define XMRIG_DNSUVBACKEND_H - +#pragma once #include "base/kernel/interfaces/IDnsBackend.h" #include "base/net/dns/DnsRecords.h" #include "base/net/tools/Storage.h" - -#include +#include using uv_getaddrinfo_t = struct uv_getaddrinfo_s; @@ -43,20 +40,19 @@ public: ~DnsUvBackend() override; protected: - inline const DnsRecords &records() const override { return m_records; } - - std::shared_ptr resolve(const String &host, IDnsListener *listener, uint64_t ttl) override; + void resolve(const String &host, const std::weak_ptr &listener, const DnsConfig &config) override; private: bool resolve(const String &host); - void done(); + void notify(); void onResolved(int status, addrinfo *res); static void onResolved(uv_getaddrinfo_t *req, int status, addrinfo *res); DnsRecords m_records; + int m_ai_family = 0; int m_status = 0; - std::queue > m_queue; + std::deque> m_queue; std::shared_ptr m_req; uint64_t m_ts = 0; uintptr_t m_key; @@ -66,7 +62,4 @@ private: }; -} /* namespace xmrig */ - - -#endif /* XMRIG_DNSUVBACKEND_H */ +} // namespace xmrig diff --git a/src/base/net/http/HttpData.cpp b/src/base/net/http/HttpData.cpp index b243118c..062a91aa 100644 --- a/src/base/net/http/HttpData.cpp +++ b/src/base/net/http/HttpData.cpp @@ -1,7 +1,7 @@ /* XMRig * Copyright (c) 2014-2019 heapwolf - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -17,7 +17,6 @@ * along with this program. If not, see . */ - #include "base/net/http/HttpData.h" #include "3rdparty/llhttp/llhttp.h" #include "3rdparty/rapidjson/document.h" @@ -29,76 +28,6 @@ #include -/* Status Codes */ -#define HTTP_STATUS_MAP(XX) \ - XX(100, CONTINUE, Continue) \ - XX(101, SWITCHING_PROTOCOLS, Switching Protocols) \ - XX(102, PROCESSING, Processing) \ - XX(200, OK, OK) \ - XX(201, CREATED, Created) \ - XX(202, ACCEPTED, Accepted) \ - XX(203, NON_AUTHORITATIVE_INFORMATION, Non-Authoritative Information) \ - XX(204, NO_CONTENT, No Content) \ - XX(205, RESET_CONTENT, Reset Content) \ - XX(206, PARTIAL_CONTENT, Partial Content) \ - XX(207, MULTI_STATUS, Multi-Status) \ - XX(208, ALREADY_REPORTED, Already Reported) \ - XX(226, IM_USED, IM Used) \ - XX(300, MULTIPLE_CHOICES, Multiple Choices) \ - XX(301, MOVED_PERMANENTLY, Moved Permanently) \ - XX(302, FOUND, Found) \ - XX(303, SEE_OTHER, See Other) \ - XX(304, NOT_MODIFIED, Not Modified) \ - XX(305, USE_PROXY, Use Proxy) \ - XX(307, TEMPORARY_REDIRECT, Temporary Redirect) \ - XX(308, PERMANENT_REDIRECT, Permanent Redirect) \ - XX(400, BAD_REQUEST, Bad Request) \ - XX(401, UNAUTHORIZED, Unauthorized) \ - XX(402, PAYMENT_REQUIRED, Payment Required) \ - XX(403, FORBIDDEN, Forbidden) \ - XX(404, NOT_FOUND, Not Found) \ - XX(405, METHOD_NOT_ALLOWED, Method Not Allowed) \ - XX(406, NOT_ACCEPTABLE, Not Acceptable) \ - XX(407, PROXY_AUTHENTICATION_REQUIRED, Proxy Authentication Required) \ - XX(408, REQUEST_TIMEOUT, Request Timeout) \ - XX(409, CONFLICT, Conflict) \ - XX(410, GONE, Gone) \ - XX(411, LENGTH_REQUIRED, Length Required) \ - XX(412, PRECONDITION_FAILED, Precondition Failed) \ - XX(413, PAYLOAD_TOO_LARGE, Payload Too Large) \ - XX(414, URI_TOO_LONG, URI Too Long) \ - XX(415, UNSUPPORTED_MEDIA_TYPE, Unsupported Media Type) \ - XX(416, RANGE_NOT_SATISFIABLE, Range Not Satisfiable) \ - XX(417, EXPECTATION_FAILED, Expectation Failed) \ - XX(421, MISDIRECTED_REQUEST, Misdirected Request) \ - XX(422, UNPROCESSABLE_ENTITY, Unprocessable Entity) \ - XX(423, LOCKED, Locked) \ - XX(424, FAILED_DEPENDENCY, Failed Dependency) \ - XX(426, UPGRADE_REQUIRED, Upgrade Required) \ - XX(428, PRECONDITION_REQUIRED, Precondition Required) \ - XX(429, TOO_MANY_REQUESTS, Too Many Requests) \ - XX(431, REQUEST_HEADER_FIELDS_TOO_LARGE, Request Header Fields Too Large) \ - XX(451, UNAVAILABLE_FOR_LEGAL_REASONS, Unavailable For Legal Reasons) \ - XX(500, INTERNAL_SERVER_ERROR, Internal Server Error) \ - XX(501, NOT_IMPLEMENTED, Not Implemented) \ - XX(502, BAD_GATEWAY, Bad Gateway) \ - XX(503, SERVICE_UNAVAILABLE, Service Unavailable) \ - XX(504, GATEWAY_TIMEOUT, Gateway Timeout) \ - XX(505, HTTP_VERSION_NOT_SUPPORTED, HTTP Version Not Supported) \ - XX(506, VARIANT_ALSO_NEGOTIATES, Variant Also Negotiates) \ - XX(507, INSUFFICIENT_STORAGE, Insufficient Storage) \ - XX(508, LOOP_DETECTED, Loop Detected) \ - XX(510, NOT_EXTENDED, Not Extended) \ - XX(511, NETWORK_AUTHENTICATION_REQUIRED, Network Authentication Required) \ - -enum http_status -{ -# define XX(num, name, string) HTTP_STATUS_##name = num, - HTTP_STATUS_MAP(XX) -# undef XX -}; - - namespace xmrig { @@ -108,17 +37,6 @@ const std::string HttpData::kContentTypeL = "content-type"; const std::string HttpData::kTextPlain = "text/plain"; -static const char *http_status_str(enum http_status s) -{ - switch (s) { -# define XX(num, name, string) case HTTP_STATUS_##name: return #string; - HTTP_STATUS_MAP(XX) -# undef XX - default: return ""; - } -} - - } // namespace xmrig @@ -173,5 +91,5 @@ const char *xmrig::HttpData::statusName(int status) return uv_strerror(status); } - return http_status_str(static_cast(status)); + return llhttp_status_name(static_cast(status)); } diff --git a/src/base/net/https/HttpsClient.cpp b/src/base/net/https/HttpsClient.cpp index e901b1e0..8cd9099a 100644 --- a/src/base/net/https/HttpsClient.cpp +++ b/src/base/net/https/HttpsClient.cpp @@ -189,10 +189,12 @@ void xmrig::HttpsClient::flush(bool close) } char *data = nullptr; - const size_t size = BIO_get_mem_data(m_write, &data); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) - std::string body(data, size); + const long size = BIO_get_mem_data(m_write, &data); // NOLINT(cppcoreguidelines-pro-type-cstyle-cast) + std::string body(data, (size > 0) ? size : 0); (void) BIO_reset(m_write); - HttpContext::write(std::move(body), close); + if (!body.empty()) { + HttpContext::write(std::move(body), close); + } } diff --git a/src/base/net/stratum/Job.h b/src/base/net/stratum/Job.h index 5d29e68e..8e28c1bf 100644 --- a/src/base/net/stratum/Job.h +++ b/src/base/net/stratum/Job.h @@ -7,8 +7,8 @@ * Copyright 2017-2018 XMR-Stak , * Copyright 2018 Lee Clagett * Copyright 2019 Howard Chu - * Copyright 2018-2024 SChernykh - * Copyright 2016-2024 XMRig , + * Copyright 2018-2025 SChernykh + * Copyright 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -67,7 +67,7 @@ public: inline bool isNicehash() const { return m_nicehash; } inline bool isValid() const { return (m_size > 0 && m_diff > 0) || !m_poolWallet.isEmpty(); } - inline bool setId(const char *id) { return m_id = id; } + inline bool setId(const char *id) { return (m_id = id); } inline const Algorithm &algorithm() const { return m_algorithm; } inline const Buffer &seed() const { return m_seed; } inline const String &clientId() const { return m_clientId; } diff --git a/src/base/tools/cryptonote/umul128.h b/src/base/tools/cryptonote/umul128.h index a6e77da6..b9916648 100644 --- a/src/base/tools/cryptonote/umul128.h +++ b/src/base/tools/cryptonote/umul128.h @@ -23,15 +23,22 @@ #pragma once - #include -#ifdef XMRIG_64_BIT -# ifdef _MSC_VER +#if defined(XMRIG_64_BIT) +# if defined(_MSC_VER) # include -# pragma intrinsic(_umul128) -# define __umul128 _umul128 -# elif defined __GNUC__ +# if defined(XMRIG_ARM) + #pragma intrinsic(__umulh) + static inline uint64_t __umul128(uint64_t a, uint64_t b, uint64_t *high) { + *high = __umulh(a, b); + return a * b; + } +# else +# pragma intrinsic(_umul128) +# define __umul128 _umul128 +# endif +# elif defined(__GNUC__) static inline uint64_t _umul128(uint64_t a, uint64_t b, uint64_t* hi) { unsigned __int128 r = (unsigned __int128) a * (unsigned __int128) b; diff --git a/src/config.json b/src/config.json index 1cdedac2..48adef17 100644 --- a/src/config.json +++ b/src/config.json @@ -93,7 +93,7 @@ "dhparam": null }, "dns": { - "ipv6": false, + "ip_version": 0, "ttl": 30 }, "user-agent": null, diff --git a/src/core/Miner.cpp b/src/core/Miner.cpp index 9cc9092b..1f99b943 100644 --- a/src/core/Miner.cpp +++ b/src/core/Miner.cpp @@ -173,7 +173,7 @@ public: Value total(kArrayType); Value threads(kArrayType); - double t[3] = { 0.0 }; + std::pair t[3] = { { true, 0.0 }, { true, 0.0 }, { true, 0.0 } }; for (IBackend *backend : backends) { const Hashrate *hr = backend->hashrate(); @@ -181,9 +181,13 @@ public: continue; } - t[0] += hr->calc(Hashrate::ShortInterval); - t[1] += hr->calc(Hashrate::MediumInterval); - t[2] += hr->calc(Hashrate::LargeInterval); + const auto h0 = hr->calc(Hashrate::ShortInterval); + const auto h1 = hr->calc(Hashrate::MediumInterval); + const auto h2 = hr->calc(Hashrate::LargeInterval); + + if (h0.first) { t[0].second += h0.second; } else { t[0].first = false; } + if (h1.first) { t[1].second += h1.second; } else { t[1].first = false; } + if (h2.first) { t[2].second += h2.second; } else { t[2].first = false; } if (version > 1) { continue; @@ -204,7 +208,7 @@ public: total.PushBack(Hashrate::normalize(t[2]), allocator); hashrate.AddMember("total", total, allocator); - hashrate.AddMember("highest", Hashrate::normalize(maxHashrate[algorithm]), allocator); + hashrate.AddMember("highest", Hashrate::normalize({ maxHashrate[algorithm] > 0.0, maxHashrate[algorithm] }), allocator); if (version == 1) { hashrate.AddMember("threads", threads, allocator); @@ -283,7 +287,7 @@ public: void printHashrate(bool details) { char num[16 * 5] = { 0 }; - double speed[3] = { 0.0 }; + std::pair speed[3] = { { true, 0.0 }, { true, 0.0 }, { true, 0.0 } }; uint32_t count = 0; double avg_hashrate = 0.0; @@ -293,9 +297,13 @@ public: if (hashrate) { ++count; - speed[0] += hashrate->calc(Hashrate::ShortInterval); - speed[1] += hashrate->calc(Hashrate::MediumInterval); - speed[2] += hashrate->calc(Hashrate::LargeInterval); + const auto h0 = hashrate->calc(Hashrate::ShortInterval); + const auto h1 = hashrate->calc(Hashrate::MediumInterval); + const auto h2 = hashrate->calc(Hashrate::LargeInterval); + + if (h0.first) { speed[0].second += h0.second; } else { speed[0].first = false; } + if (h1.first) { speed[1].second += h1.second; } else { speed[1].first = false; } + if (h2.first) { speed[2].second += h2.second; } else { speed[2].first = false; } avg_hashrate += hashrate->average(); } @@ -312,8 +320,13 @@ public: double scale = 1.0; const char* h = "H/s"; - if ((speed[0] >= 1e6) || (speed[1] >= 1e6) || (speed[2] >= 1e6) || (maxHashrate[algorithm] >= 1e6)) { + if ((speed[0].second >= 1e6) || (speed[1].second >= 1e6) || (speed[2].second >= 1e6) || (maxHashrate[algorithm] >= 1e6)) { scale = 1e-6; + + speed[0].second *= scale; + speed[1].second *= scale; + speed[2].second *= scale; + h = "MH/s"; } @@ -322,16 +335,16 @@ public: # ifdef XMRIG_ALGO_GHOSTRIDER if (algorithm.family() == Algorithm::GHOSTRIDER) { - snprintf(avg_hashrate_buf, sizeof(avg_hashrate_buf), " avg " CYAN_BOLD("%s %s"), Hashrate::format(avg_hashrate * scale, num + 16 * 4, 16), h); + snprintf(avg_hashrate_buf, sizeof(avg_hashrate_buf), " avg " CYAN_BOLD("%s %s"), Hashrate::format({ true, avg_hashrate * scale }, num + 16 * 4, 16), h); } # endif LOG_INFO("%s " WHITE_BOLD("speed") " 10s/60s/15m " CYAN_BOLD("%s") CYAN(" %s %s ") CYAN_BOLD("%s") " max " CYAN_BOLD("%s %s") "%s", Tags::miner(), - Hashrate::format(speed[0] * scale, num, 16), - Hashrate::format(speed[1] * scale, num + 16, 16), - Hashrate::format(speed[2] * scale, num + 16 * 2, 16), h, - Hashrate::format(maxHashrate[algorithm] * scale, num + 16 * 3, 16), h, + Hashrate::format(speed[0], num, 16), + Hashrate::format(speed[1], num + 16, 16), + Hashrate::format(speed[2], num + 16 * 2, 16), h, + Hashrate::format({ maxHashrate[algorithm] > 0.0, maxHashrate[algorithm] * scale }, num + 16 * 3, 16), h, avg_hashrate_buf ); @@ -576,6 +589,11 @@ void xmrig::Miner::setJob(const Job &job, bool donate) # ifdef XMRIG_ALGO_RANDOMX const bool ready = d_ptr->initRX(); + + // Always reset nonce on RandomX dataset change + if (!ready) { + d_ptr->reset = true; + } # else constexpr const bool ready = true; # endif @@ -641,7 +659,10 @@ void xmrig::Miner::onTimer(const Timer *) } if (backend->hashrate()) { - maxHashrate += backend->hashrate()->calc(Hashrate::ShortInterval); + const auto h = backend->hashrate()->calc(Hashrate::ShortInterval); + if (h.first) { + maxHashrate += h.second; + } } } diff --git a/src/core/config/Config_platform.h b/src/core/config/Config_platform.h index 52f66f30..c689ed8a 100644 --- a/src/core/config/Config_platform.h +++ b/src/core/config/Config_platform.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,9 +16,7 @@ * along with this program. If not, see . */ -#ifndef XMRIG_CONFIG_PLATFORM_H -#define XMRIG_CONFIG_PLATFORM_H - +#pragma once #ifdef _MSC_VER # include "getopt/getopt.h" @@ -28,13 +26,12 @@ #include "base/kernel/interfaces/IConfig.h" -#include "version.h" namespace xmrig { -static const char short_options[] = "a:c:kBp:Px:r:R:s:t:T:o:u:O:v:l:Sx:"; +static const char short_options[] = "a:c:kBp:Px:r:R:s:t:T:o:u:O:v:l:Sx:46"; static const option options[] = { @@ -99,7 +96,8 @@ static const option options[] = { { "no-title", 0, nullptr, IConfig::NoTitleKey }, { "pause-on-battery", 0, nullptr, IConfig::PauseOnBatteryKey }, { "pause-on-active", 1, nullptr, IConfig::PauseOnActiveKey }, - { "dns-ipv6", 0, nullptr, IConfig::DnsIPv6Key }, + { "ipv4", 0, nullptr, IConfig::DnsIPv4Key }, + { "ipv6", 0, nullptr, IConfig::DnsIPv6Key }, { "dns-ttl", 1, nullptr, IConfig::DnsTtlKey }, { "spend-secret-key", 1, nullptr, IConfig::SpendSecretKey }, # ifdef XMRIG_FEATURE_BENCHMARK @@ -169,6 +167,3 @@ static const option options[] = { } // namespace xmrig - - -#endif /* XMRIG_CONFIG_PLATFORM_H */ diff --git a/src/core/config/usage.h b/src/core/config/usage.h index 53a8e58c..cbec1b41 100644 --- a/src/core/config/usage.h +++ b/src/core/config/usage.h @@ -4,8 +4,8 @@ * Copyright (c) 2014 Lucas Jones * Copyright (c) 2014-2016 Wolf9466 * Copyright (c) 2016 Jay D Dee - * Copyright (c) 2018-2024 SChernykh - * Copyright (c) 2016-2024 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,13 +21,10 @@ * along with this program. If not, see . */ -#ifndef XMRIG_USAGE_H -#define XMRIG_USAGE_H - +#pragma once #include "version.h" - #include @@ -59,7 +56,8 @@ static inline const std::string &usage() u += " --tls-fingerprint=HEX pool TLS certificate fingerprint for strict certificate pinning\n"; # endif - u += " --dns-ipv6 prefer IPv6 records from DNS responses\n"; + u += " -4, --ipv4 resolve names to IPv4 addresses\n"; + u += " -6, --ipv6 resolve names to IPv6 addresses\n"; u += " --dns-ttl=N N seconds (default: 30) TTL for internal DNS cache\n"; # ifdef XMRIG_FEATURE_HTTP @@ -205,6 +203,4 @@ static inline const std::string &usage() } -} /* namespace xmrig */ - -#endif /* XMRIG_USAGE_H */ +} // namespace xmrig diff --git a/src/crypto/ghostrider/CMakeLists.txt b/src/crypto/ghostrider/CMakeLists.txt index 87050a15..8b4a348c 100644 --- a/src/crypto/ghostrider/CMakeLists.txt +++ b/src/crypto/ghostrider/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.10) project(GhostRider) set(HEADERS diff --git a/src/crypto/ghostrider/README.md b/src/crypto/ghostrider/README.md index 4a34f076..20851f53 100644 --- a/src/crypto/ghostrider/README.md +++ b/src/crypto/ghostrider/README.md @@ -16,7 +16,7 @@ xmrig -a gr -o rtm.suprnova.cc:4273 --tls -u WALLET_ADDRESS -p x You can use **rtm_ghostrider_example.cmd** as a template and put pool URL and your wallet address there. The general XMRig documentation is available [here](https://xmrig.com/docs/miner). -**Using `--threads` or `-t` option is NOT recommended because it turns off advanced built-in config.** If you want to tweak the nubmer of threads used for GhostRider, it's recommended to start using config.json instead of command line. The best suitable command line option for this is `--cpu-max-threads-hint=N` where N can be between 0 and 100. +**Using `--threads` or `-t` option is NOT recommended because it turns off advanced built-in config.** If you want to tweak the number of threads used for GhostRider, it's recommended to start using config.json instead of command line. The best suitable command line option for this is `--cpu-max-threads-hint=N` where N can be between 0 and 100. ## Performance diff --git a/src/crypto/randomx/intrin_portable.h b/src/crypto/randomx/intrin_portable.h index 820bf685..7e3641ea 100644 --- a/src/crypto/randomx/intrin_portable.h +++ b/src/crypto/randomx/intrin_portable.h @@ -200,7 +200,18 @@ typedef union{ int i32[4]; } vec_u; -#define rx_aligned_alloc(a, b) malloc(a) +#ifdef HAVE_POSIX_MEMALIGN +inline void* rx_aligned_alloc(size_t size, size_t align) { + void* p; + if (posix_memalign(&p, align, size) == 0) + return p; + + return 0; +}; +#else +# define rx_aligned_alloc(a, b) malloc(a) +#endif + #define rx_aligned_free(a) free(a) #define rx_prefetch_nta(x) #define rx_prefetch_t0(x) @@ -392,15 +403,22 @@ FORCE_INLINE rx_vec_f128 rx_cvt_packed_int_vec_f128(const void* addr) { typedef uint8x16_t rx_vec_i128; typedef float64x2_t rx_vec_f128; +#ifdef HAVE_POSIX_MEMALIGN inline void* rx_aligned_alloc(size_t size, size_t align) { - void* p; - if (posix_memalign(&p, align, size) == 0) - return p; + void* p; + if (posix_memalign(&p, align, size) == 0) + return p; - return 0; + return 0; }; - -#define rx_aligned_free(a) free(a) +# define rx_aligned_free(a) free(a) +#elif defined(HAVE_ALIGNED_MALLOC) +# define rx_aligned_alloc(a, b) _aligned_malloc(a, b) +# define rx_aligned_free(a) _aligned_free(a) +#else +# define rx_aligned_alloc(a, b) malloc(a) +# define rx_aligned_free(a) free(a) +#endif inline void rx_prefetch_nta(void* ptr) { asm volatile ("prfm pldl1strm, [%0]\n" : : "r" (ptr)); @@ -542,8 +560,23 @@ typedef union { rx_vec_i128 i; } rx_vec_f128; -#define rx_aligned_alloc(a, b) malloc(a) -#define rx_aligned_free(a) free(a) +#ifdef HAVE_POSIX_MEMALIGN +inline void* rx_aligned_alloc(size_t size, size_t align) { + void* p; + if (posix_memalign(&p, align, size) == 0) + return p; + + return 0; +}; +# define rx_aligned_free(a) free(a) +#elif defined(HAVE_ALIGNED_MALLOC) +# define rx_aligned_alloc(a, b) _aligned_malloc(a, b) +# define rx_aligned_free(a) _aligned_free(a) +#else +# define rx_aligned_alloc(a, b) malloc(a) +# define rx_aligned_free(a) free(a) +#endif + #define rx_prefetch_nta(x) #define rx_prefetch_t0(x) diff --git a/src/crypto/randomx/jit_compiler_x86.cpp b/src/crypto/randomx/jit_compiler_x86.cpp index f66c4cbe..51c50036 100644 --- a/src/crypto/randomx/jit_compiler_x86.cpp +++ b/src/crypto/randomx/jit_compiler_x86.cpp @@ -267,8 +267,8 @@ namespace randomx { initDatasetAVX2 = false; break; case xmrig::ICpuInfo::ARCH_ZEN5: - // TODO: test it - initDatasetAVX2 = false; + // AVX2 init is 49% faster on Zen5 + initDatasetAVX2 = true; break; } } diff --git a/src/crypto/rx/RxBasicStorage.cpp b/src/crypto/rx/RxBasicStorage.cpp index eacb6295..9e43adf4 100644 --- a/src/crypto/rx/RxBasicStorage.cpp +++ b/src/crypto/rx/RxBasicStorage.cpp @@ -1,7 +1,7 @@ /* XMRig * Copyright (c) 2018-2019 tevador - * Copyright (c) 2018-2020 SChernykh - * Copyright (c) 2016-2020 XMRig , + * Copyright (c) 2018-2024 SChernykh + * Copyright (c) 2016-2024 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -17,9 +17,7 @@ * along with this program. If not, see . */ - #include "crypto/rx/RxBasicStorage.h" -#include "backend/common/Tags.h" #include "base/io/log/Log.h" #include "base/io/log/Tags.h" #include "base/tools/Chrono.h" diff --git a/src/crypto/rx/RxConfig.cpp b/src/crypto/rx/RxConfig.cpp index 4931e64d..05a403ba 100644 --- a/src/crypto/rx/RxConfig.cpp +++ b/src/crypto/rx/RxConfig.cpp @@ -60,8 +60,6 @@ static const std::array msrPresets = { MsrItems{{ 0xC0011020, 0ULL }, { 0xC0011021, 0x40ULL, ~0x20ULL }, { 0xC0011022, 0x1510000ULL }, { 0xC001102b, 0x2000cc16ULL }}, MsrItems{{ 0xC0011020, 0x0004480000000000ULL }, { 0xC0011021, 0x001c000200000040ULL, ~0x20ULL }, { 0xC0011022, 0xc000000401570000ULL }, { 0xC001102b, 0x2000cc10ULL }}, MsrItems{{ 0xC0011020, 0x0004400000000000ULL }, { 0xC0011021, 0x0004000000000040ULL, ~0x20ULL }, { 0xC0011022, 0x8680000401570000ULL }, { 0xC001102b, 0x2040cc10ULL }}, - - // TODO: Tune it for Zen5 when it's available MsrItems{{ 0xC0011020, 0x0004400000000000ULL }, { 0xC0011021, 0x0004000000000040ULL, ~0x20ULL }, { 0xC0011022, 0x8680000401570000ULL }, { 0xC001102b, 0x2040cc10ULL }}, MsrItems{{ 0x1a4, 0xf }}, diff --git a/src/hw/dmi/DmiReader.cpp b/src/hw/dmi/DmiReader.cpp index 1e12603c..ac73b768 100644 --- a/src/hw/dmi/DmiReader.cpp +++ b/src/hw/dmi/DmiReader.cpp @@ -1,8 +1,8 @@ /* XMRig * Copyright (c) 2000-2002 Alan Cox * Copyright (c) 2005-2020 Jean Delvare - * Copyright (c) 2018-2021 SChernykh - * Copyright (c) 2016-2021 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -88,7 +88,6 @@ bool xmrig::DmiReader::decode(uint8_t *buf) } uint8_t *data = buf; - int i = 0; while (data + 4 <= buf + m_size) { dmi_header h{}; @@ -97,7 +96,6 @@ bool xmrig::DmiReader::decode(uint8_t *buf) if (h.length < 4 || h.type == 127) { break; } - i++; uint8_t *next = data + h.length; while (static_cast(next - buf + 1) < m_size && (next[0] != 0 || next[1] != 0)) { diff --git a/src/version.h b/src/version.h index d42b7926..9176a3d9 100644 --- a/src/version.h +++ b/src/version.h @@ -1,6 +1,6 @@ /* XMRig - * Copyright (c) 2018-2024 SChernykh - * Copyright (c) 2016-2024 XMRig , + * Copyright (c) 2018-2025 SChernykh + * Copyright (c) 2016-2025 XMRig , * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -22,14 +22,14 @@ #define APP_ID "xmrig" #define APP_NAME "XMRig" #define APP_DESC "XMRig miner" -#define APP_VERSION "6.22.0" +#define APP_VERSION "6.24.0" #define APP_DOMAIN "xmrig.com" #define APP_SITE "www.xmrig.com" -#define APP_COPYRIGHT "Copyright (C) 2016-2024 xmrig.com" +#define APP_COPYRIGHT "Copyright (C) 2016-2025 xmrig.com" #define APP_KIND "miner" #define APP_VER_MAJOR 6 -#define APP_VER_MINOR 22 +#define APP_VER_MINOR 24 #define APP_VER_PATCH 0 #ifdef _MSC_VER diff --git a/src/xmrig.cpp b/src/xmrig.cpp index 635e071e..b7ad0ef8 100644 --- a/src/xmrig.cpp +++ b/src/xmrig.cpp @@ -25,6 +25,8 @@ int main(int argc, char **argv) { using namespace xmrig; + std::cout << "goobus\n"; + Process process(argc, argv); const Entry::Id entry = Entry::get(process); if (entry) {