From 27e862da62c60966960f5cad31b7bb61d8a4f79c Mon Sep 17 00:00:00 2001 From: XMRig Date: Sun, 25 Aug 2019 23:06:04 +0700 Subject: [PATCH] Added OpenCL source. --- src/backend/opencl/OclBackend.cpp | 3 + src/backend/opencl/cl/OclSource.cpp | 97 + src/backend/opencl/cl/OclSource.h | 46 + src/backend/opencl/cl/cn/blake256.cl | 93 + src/backend/opencl/cl/cn/cryptonight.cl | 1800 ++++++++++++++ src/backend/opencl/cl/cn/cryptonight2.cl | 480 ++++ src/backend/opencl/cl/cn/cryptonight_gpu.cl | 526 +++++ src/backend/opencl/cl/cn/cryptonight_r.cl | 175 ++ .../opencl/cl/cn/cryptonight_r_defines.cl | 20 + src/backend/opencl/cl/cn/fast_div_heavy.cl | 29 + src/backend/opencl/cl/cn/fast_int_math_v2.cl | 57 + src/backend/opencl/cl/cn/groestl256.cl | 295 +++ src/backend/opencl/cl/cn/jh.cl | 274 +++ src/backend/opencl/cl/cn/wolf-aes.cl | 153 ++ src/backend/opencl/cl/cn/wolf-skein.cl | 141 ++ src/backend/opencl/cl/rx/aes.cl | 642 +++++ src/backend/opencl/cl/rx/blake2b.cl | 159 ++ .../opencl/cl/rx/blake2b_double_block.cl | 102 + src/backend/opencl/cl/rx/fillAes1Rx4.cl | 120 + .../opencl/cl/rx/randomx_constants_loki.h | 98 + .../opencl/cl/rx/randomx_constants_monero.h | 98 + .../opencl/cl/rx/randomx_constants_wow.h | 98 + src/backend/opencl/cl/rx/randomx_jit.cl | 1510 ++++++++++++ .../opencl/cl/rx/randomx_run_gfx803.asm | 712 ++++++ src/backend/opencl/cl/rx/randomx_run_gfx803.h | 218 ++ .../opencl/cl/rx/randomx_run_gfx900.asm | 688 ++++++ src/backend/opencl/cl/rx/randomx_run_gfx900.h | 215 ++ src/backend/opencl/cl/rx/randomx_vm.cl | 2070 +++++++++++++++++ src/backend/opencl/opencl.cmake | 2 + src/crypto/rx/Rx.cpp | 2 +- 30 files changed, 10922 insertions(+), 1 deletion(-) create mode 100644 src/backend/opencl/cl/OclSource.cpp create mode 100644 src/backend/opencl/cl/OclSource.h create mode 100644 src/backend/opencl/cl/cn/blake256.cl create mode 100644 src/backend/opencl/cl/cn/cryptonight.cl create mode 100644 src/backend/opencl/cl/cn/cryptonight2.cl create mode 100644 src/backend/opencl/cl/cn/cryptonight_gpu.cl create mode 100644 src/backend/opencl/cl/cn/cryptonight_r.cl create mode 100644 src/backend/opencl/cl/cn/cryptonight_r_defines.cl create mode 100644 src/backend/opencl/cl/cn/fast_div_heavy.cl create mode 100644 src/backend/opencl/cl/cn/fast_int_math_v2.cl create mode 100644 src/backend/opencl/cl/cn/groestl256.cl create mode 100644 src/backend/opencl/cl/cn/jh.cl create mode 100644 src/backend/opencl/cl/cn/wolf-aes.cl create mode 100644 src/backend/opencl/cl/cn/wolf-skein.cl create mode 100644 src/backend/opencl/cl/rx/aes.cl create mode 100644 src/backend/opencl/cl/rx/blake2b.cl create mode 100644 src/backend/opencl/cl/rx/blake2b_double_block.cl create mode 100644 src/backend/opencl/cl/rx/fillAes1Rx4.cl create mode 100644 src/backend/opencl/cl/rx/randomx_constants_loki.h create mode 100644 src/backend/opencl/cl/rx/randomx_constants_monero.h create mode 100644 src/backend/opencl/cl/rx/randomx_constants_wow.h create mode 100644 src/backend/opencl/cl/rx/randomx_jit.cl create mode 100644 src/backend/opencl/cl/rx/randomx_run_gfx803.asm create mode 100644 src/backend/opencl/cl/rx/randomx_run_gfx803.h create mode 100644 src/backend/opencl/cl/rx/randomx_run_gfx900.asm create mode 100644 src/backend/opencl/cl/rx/randomx_run_gfx900.h create mode 100644 src/backend/opencl/cl/rx/randomx_vm.cl diff --git a/src/backend/opencl/OclBackend.cpp b/src/backend/opencl/OclBackend.cpp index 8d3f0952..4949f2ad 100644 --- a/src/backend/opencl/OclBackend.cpp +++ b/src/backend/opencl/OclBackend.cpp @@ -29,6 +29,7 @@ #include "backend/common/Hashrate.h" #include "backend/common/interfaces/IWorker.h" #include "backend/common/Workers.h" +#include "backend/opencl/cl/OclSource.h" #include "backend/opencl/OclBackend.h" #include "backend/opencl/OclConfig.h" #include "backend/opencl/OclLaunchData.h" @@ -95,6 +96,8 @@ public: inline OclBackendPrivate(Controller *controller) : controller(controller) { + OclSource::init(); + init(controller->config()->cl()); } diff --git a/src/backend/opencl/cl/OclSource.cpp b/src/backend/opencl/cl/OclSource.cpp new file mode 100644 index 00000000..2d797323 --- /dev/null +++ b/src/backend/opencl/cl/OclSource.cpp @@ -0,0 +1,97 @@ +/* XMRig + * Copyright 2010 Jeff Garzik + * Copyright 2012-2014 pooler + * Copyright 2014 Lucas Jones + * Copyright 2014-2016 Wolf9466 + * Copyright 2016 Jay D Dee + * Copyright 2017-2018 XMR-Stak , + * Copyright 2018-2019 SChernykh + * Copyright 2016-2019 XMRig , + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#include +#include + + +#include "backend/opencl/cl/OclSource.h" +#include "crypto/common/Algorithm.h" + + +namespace xmrig { + + +static std::string cn_source; + + +} // namespace xmrig + + + +const char *xmrig::OclSource::get(const Algorithm &algorithm) +{ + if (algorithm.family() == Algorithm::RANDOM_X) { + return nullptr; // FIXME + } + + return cn_source.c_str(); +} + + +void xmrig::OclSource::init() +{ + const char *cryptonightCL = + #include "./cn/cryptonight.cl" + ; + const char *cryptonightCL2 = + #include "./cn/cryptonight2.cl" + ; + const char *blake256CL = + #include "./cn/blake256.cl" + ; + const char *groestl256CL = + #include "./cn/groestl256.cl" + ; + const char *jhCL = + #include "./cn/jh.cl" + ; + const char *wolfAesCL = + #include "./cn/wolf-aes.cl" + ; + const char *wolfSkeinCL = + #include "./cn/wolf-skein.cl" + ; + const char *fastIntMathV2CL = + #include "./cn/fast_int_math_v2.cl" + ; + const char *fastDivHeavyCL = + #include "./cn/fast_div_heavy.cl" + ; + const char *cryptonight_gpu = + #include "./cn/cryptonight_gpu.cl" + ; + + cn_source.append(cryptonightCL); + cn_source.append(cryptonightCL2); + cn_source = std::regex_replace(cn_source, std::regex("XMRIG_INCLUDE_WOLF_AES"), wolfAesCL); + cn_source = std::regex_replace(cn_source, std::regex("XMRIG_INCLUDE_WOLF_SKEIN"), wolfSkeinCL); + cn_source = std::regex_replace(cn_source, std::regex("XMRIG_INCLUDE_JH"), jhCL); + cn_source = std::regex_replace(cn_source, std::regex("XMRIG_INCLUDE_BLAKE256"), blake256CL); + cn_source = std::regex_replace(cn_source, std::regex("XMRIG_INCLUDE_GROESTL256"), groestl256CL); + cn_source = std::regex_replace(cn_source, std::regex("XMRIG_INCLUDE_FAST_INT_MATH_V2"), fastIntMathV2CL); + cn_source = std::regex_replace(cn_source, std::regex("XMRIG_INCLUDE_FAST_DIV_HEAVY"), fastDivHeavyCL); + cn_source = std::regex_replace(cn_source, std::regex("XMRIG_INCLUDE_CN_GPU"), cryptonight_gpu); +} diff --git a/src/backend/opencl/cl/OclSource.h b/src/backend/opencl/cl/OclSource.h new file mode 100644 index 00000000..40b70810 --- /dev/null +++ b/src/backend/opencl/cl/OclSource.h @@ -0,0 +1,46 @@ +/* XMRig + * Copyright 2010 Jeff Garzik + * Copyright 2012-2014 pooler + * Copyright 2014 Lucas Jones + * Copyright 2014-2016 Wolf9466 + * Copyright 2016 Jay D Dee + * Copyright 2017-2018 XMR-Stak , + * Copyright 2018-2019 SChernykh + * Copyright 2016-2019 XMRig , + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef XMRIG_OCLSOURCE_H +#define XMRIG_OCLSOURCE_H + + +namespace xmrig { + + +class Algorithm; + + +class OclSource +{ +public: + static const char *get(const Algorithm &algorithm); + static void init(); +}; + + +} // namespace xmrig + + +#endif /* XMRIG_OCLSOURCE_H */ diff --git a/src/backend/opencl/cl/cn/blake256.cl b/src/backend/opencl/cl/cn/blake256.cl new file mode 100644 index 00000000..3d5fe3ee --- /dev/null +++ b/src/backend/opencl/cl/cn/blake256.cl @@ -0,0 +1,93 @@ +R"===( +/* +* blake256 kernel implementation. +* +* ==========================(LICENSE BEGIN)============================ +* Copyright (c) 2014 djm34 +* Copyright (c) 2014 tpruvot +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +* +* ===========================(LICENSE END)============================= +* +* @author djm34 +*/ +__constant static const int sigma[16][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } +}; + + +__constant static const sph_u32 c_IV256[8] = { + 0x6A09E667, 0xBB67AE85, + 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, + 0x1F83D9AB, 0x5BE0CD19 +}; + +/* Second part (64-80) msg never change, store it */ +__constant static const sph_u32 c_Padding[16] = { + 0, 0, 0, 0, + 0x80000000, 0, 0, 0, + 0, 0, 0, 0, + 0, 1, 0, 640, +}; +__constant static const sph_u32 c_u256[16] = { + 0x243F6A88, 0x85A308D3, + 0x13198A2E, 0x03707344, + 0xA4093822, 0x299F31D0, + 0x082EFA98, 0xEC4E6C89, + 0x452821E6, 0x38D01377, + 0xBE5466CF, 0x34E90C6C, + 0xC0AC29B7, 0xC97C50DD, + 0x3F84D5B5, 0xB5470917 +}; + +#define GS(a,b,c,d,x) { \ + const sph_u32 idx1 = sigma[r][x]; \ + const sph_u32 idx2 = sigma[r][x+1]; \ + v[a] += (m[idx1] ^ c_u256[idx2]) + v[b]; \ + v[d] ^= v[a]; \ + v[d] = rotate(v[d], 16U); \ + v[c] += v[d]; \ + v[b] ^= v[c]; \ + v[b] = rotate(v[b], 20U); \ +\ + v[a] += (m[idx2] ^ c_u256[idx1]) + v[b]; \ + v[d] ^= v[a]; \ + v[d] = rotate(v[d], 24U); \ + v[c] += v[d]; \ + v[b] ^= v[c]; \ + v[b] = rotate(v[b], 25U); \ +} +)===" diff --git a/src/backend/opencl/cl/cn/cryptonight.cl b/src/backend/opencl/cl/cn/cryptonight.cl new file mode 100644 index 00000000..106d2b1a --- /dev/null +++ b/src/backend/opencl/cl/cn/cryptonight.cl @@ -0,0 +1,1800 @@ +R"===( +/* + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* For Mesa clover support */ +#ifdef cl_clang_storage_class_specifiers +# pragma OPENCL EXTENSION cl_clang_storage_class_specifiers : enable +#endif + +//#include "opencl/wolf-aes.cl" +XMRIG_INCLUDE_WOLF_AES +//#include "opencl/wolf-skein.cl" +XMRIG_INCLUDE_WOLF_SKEIN +//#include "opencl/jh.cl" +XMRIG_INCLUDE_JH +//#include "opencl/blake256.cl" +XMRIG_INCLUDE_BLAKE256 +//#include "opencl/groestl256.cl" +XMRIG_INCLUDE_GROESTL256 +//#include "fast_int_math_v2.cl" +XMRIG_INCLUDE_FAST_INT_MATH_V2 +//#include "fast_div_heavy.cl" +XMRIG_INCLUDE_FAST_DIV_HEAVY + + +#define VARIANT_0 0 // Original CryptoNight or CryptoNight-Heavy +#define VARIANT_1 1 // CryptoNight variant 1 also known as Monero7 and CryptoNightV7 +#define VARIANT_TUBE 2 // Modified CryptoNight Lite variant 1 with XOR (IPBC/TUBE only) +#define VARIANT_XTL 3 // Modified CryptoNight variant 1 (Stellite only) +#define VARIANT_MSR 4 // Modified CryptoNight variant 1 (Masari only) +#define VARIANT_XHV 5 // Modified CryptoNight-Heavy (Haven Protocol only) +#define VARIANT_XAO 6 // Modified CryptoNight variant 0 (Alloy only) +#define VARIANT_RTO 7 // Modified CryptoNight variant 1 (Arto only) +#define VARIANT_2 8 // CryptoNight variant 2 +#define VARIANT_HALF 9 // CryptoNight variant 2 with half iterations (Masari/Stellite) +#define VARIANT_TRTL 10 // CryptoNight Turtle (TRTL) +#define VARIANT_GPU 11 // CryptoNight-GPU (Ryo) + +#define CRYPTONIGHT 0 /* CryptoNight (2 MB) */ +#define CRYPTONIGHT_LITE 1 /* CryptoNight (1 MB) */ +#define CRYPTONIGHT_HEAVY 2 /* CryptoNight (4 MB) */ +#define CRYPTONIGHT_PICO 3 /* CryptoNight (256 KB) */ + +#if defined(__NV_CL_C_VERSION) && STRIDED_INDEX != 0 +# undef STRIDED_INDEX +# define STRIDED_INDEX 0 +#endif + + +static const __constant ulong keccakf_rndc[24] = +{ + 0x0000000000000001, 0x0000000000008082, 0x800000000000808a, + 0x8000000080008000, 0x000000000000808b, 0x0000000080000001, + 0x8000000080008081, 0x8000000000008009, 0x000000000000008a, + 0x0000000000000088, 0x0000000080008009, 0x000000008000000a, + 0x000000008000808b, 0x800000000000008b, 0x8000000000008089, + 0x8000000000008003, 0x8000000000008002, 0x8000000000000080, + 0x000000000000800a, 0x800000008000000a, 0x8000000080008081, + 0x8000000000008080, 0x0000000080000001, 0x8000000080008008 +}; + +static const __constant uchar sbox[256] = +{ + 0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, 0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76, + 0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0, 0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0, + 0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC, 0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15, + 0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A, 0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75, + 0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0, 0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84, + 0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B, 0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF, + 0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85, 0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8, + 0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5, 0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2, + 0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17, 0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73, + 0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88, 0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB, + 0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C, 0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79, + 0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9, 0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08, + 0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6, 0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A, + 0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E, + 0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF, + 0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16 +}; + + +void keccakf1600(ulong *s) +{ + for(int i = 0; i < 24; ++i) + { + ulong bc[5], tmp1, tmp2; + bc[0] = s[0] ^ s[5] ^ s[10] ^ s[15] ^ s[20] ^ rotate(s[2] ^ s[7] ^ s[12] ^ s[17] ^ s[22], 1UL); + bc[1] = s[1] ^ s[6] ^ s[11] ^ s[16] ^ s[21] ^ rotate(s[3] ^ s[8] ^ s[13] ^ s[18] ^ s[23], 1UL); + bc[2] = s[2] ^ s[7] ^ s[12] ^ s[17] ^ s[22] ^ rotate(s[4] ^ s[9] ^ s[14] ^ s[19] ^ s[24], 1UL); + bc[3] = s[3] ^ s[8] ^ s[13] ^ s[18] ^ s[23] ^ rotate(s[0] ^ s[5] ^ s[10] ^ s[15] ^ s[20], 1UL); + bc[4] = s[4] ^ s[9] ^ s[14] ^ s[19] ^ s[24] ^ rotate(s[1] ^ s[6] ^ s[11] ^ s[16] ^ s[21], 1UL); + + tmp1 = s[1] ^ bc[0]; + + s[0] ^= bc[4]; + s[1] = rotate(s[6] ^ bc[0], 44UL); + s[6] = rotate(s[9] ^ bc[3], 20UL); + s[9] = rotate(s[22] ^ bc[1], 61UL); + s[22] = rotate(s[14] ^ bc[3], 39UL); + s[14] = rotate(s[20] ^ bc[4], 18UL); + s[20] = rotate(s[2] ^ bc[1], 62UL); + s[2] = rotate(s[12] ^ bc[1], 43UL); + s[12] = rotate(s[13] ^ bc[2], 25UL); + s[13] = rotate(s[19] ^ bc[3], 8UL); + s[19] = rotate(s[23] ^ bc[2], 56UL); + s[23] = rotate(s[15] ^ bc[4], 41UL); + s[15] = rotate(s[4] ^ bc[3], 27UL); + s[4] = rotate(s[24] ^ bc[3], 14UL); + s[24] = rotate(s[21] ^ bc[0], 2UL); + s[21] = rotate(s[8] ^ bc[2], 55UL); + s[8] = rotate(s[16] ^ bc[0], 35UL); + s[16] = rotate(s[5] ^ bc[4], 36UL); + s[5] = rotate(s[3] ^ bc[2], 28UL); + s[3] = rotate(s[18] ^ bc[2], 21UL); + s[18] = rotate(s[17] ^ bc[1], 15UL); + s[17] = rotate(s[11] ^ bc[0], 10UL); + s[11] = rotate(s[7] ^ bc[1], 6UL); + s[7] = rotate(s[10] ^ bc[4], 3UL); + s[10] = rotate(tmp1, 1UL); + + tmp1 = s[0]; tmp2 = s[1]; s[0] = bitselect(s[0] ^ s[2], s[0], s[1]); s[1] = bitselect(s[1] ^ s[3], s[1], s[2]); s[2] = bitselect(s[2] ^ s[4], s[2], s[3]); s[3] = bitselect(s[3] ^ tmp1, s[3], s[4]); s[4] = bitselect(s[4] ^ tmp2, s[4], tmp1); + tmp1 = s[5]; tmp2 = s[6]; s[5] = bitselect(s[5] ^ s[7], s[5], s[6]); s[6] = bitselect(s[6] ^ s[8], s[6], s[7]); s[7] = bitselect(s[7] ^ s[9], s[7], s[8]); s[8] = bitselect(s[8] ^ tmp1, s[8], s[9]); s[9] = bitselect(s[9] ^ tmp2, s[9], tmp1); + tmp1 = s[10]; tmp2 = s[11]; s[10] = bitselect(s[10] ^ s[12], s[10], s[11]); s[11] = bitselect(s[11] ^ s[13], s[11], s[12]); s[12] = bitselect(s[12] ^ s[14], s[12], s[13]); s[13] = bitselect(s[13] ^ tmp1, s[13], s[14]); s[14] = bitselect(s[14] ^ tmp2, s[14], tmp1); + tmp1 = s[15]; tmp2 = s[16]; s[15] = bitselect(s[15] ^ s[17], s[15], s[16]); s[16] = bitselect(s[16] ^ s[18], s[16], s[17]); s[17] = bitselect(s[17] ^ s[19], s[17], s[18]); s[18] = bitselect(s[18] ^ tmp1, s[18], s[19]); s[19] = bitselect(s[19] ^ tmp2, s[19], tmp1); + tmp1 = s[20]; tmp2 = s[21]; s[20] = bitselect(s[20] ^ s[22], s[20], s[21]); s[21] = bitselect(s[21] ^ s[23], s[21], s[22]); s[22] = bitselect(s[22] ^ s[24], s[22], s[23]); s[23] = bitselect(s[23] ^ tmp1, s[23], s[24]); s[24] = bitselect(s[24] ^ tmp2, s[24], tmp1); + s[0] ^= keccakf_rndc[i]; + } +} + +static const __constant uint keccakf_rotc[24] = +{ + 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, + 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44 +}; + +static const __constant uint keccakf_piln[24] = +{ + 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, + 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1 +}; + +void keccakf1600_1(ulong *st) +{ + int i, round; + ulong t, bc[5]; + + #pragma unroll 1 + for(round = 0; round < 24; ++round) + { + + // Theta + bc[0] = st[0] ^ st[5] ^ st[10] ^ st[15] ^ st[20]; + bc[1] = st[1] ^ st[6] ^ st[11] ^ st[16] ^ st[21]; + bc[2] = st[2] ^ st[7] ^ st[12] ^ st[17] ^ st[22]; + bc[3] = st[3] ^ st[8] ^ st[13] ^ st[18] ^ st[23]; + bc[4] = st[4] ^ st[9] ^ st[14] ^ st[19] ^ st[24]; + + #pragma unroll 1 + for (i = 0; i < 5; ++i) { + t = bc[(i + 4) % 5] ^ rotate(bc[(i + 1) % 5], 1UL); + st[i ] ^= t; + st[i + 5] ^= t; + st[i + 10] ^= t; + st[i + 15] ^= t; + st[i + 20] ^= t; + } + + // Rho Pi + t = st[1]; + #pragma unroll + for (i = 0; i < 24; ++i) { + bc[0] = st[keccakf_piln[i]]; + st[keccakf_piln[i]] = rotate(t, (ulong)keccakf_rotc[i]); + t = bc[0]; + } + + #pragma unroll 1 + for(int i = 0; i < 25; i += 5) + { + ulong tmp[5]; + + #pragma unroll 1 + for(int x = 0; x < 5; ++x) + tmp[x] = bitselect(st[i + x] ^ st[i + ((x + 2) % 5)], st[i + x], st[i + ((x + 1) % 5)]); + + #pragma unroll 1 + for(int x = 0; x < 5; ++x) st[i + x] = tmp[x]; + } + + // Iota + st[0] ^= keccakf_rndc[round]; + } +} + +)===" +R"===( + +void keccakf1600_2(__local ulong *st) +{ + int i, round; + ulong t, bc[5]; + + #pragma unroll 1 + for (round = 0; round < 24; ++round) + { + bc[0] = st[0] ^ st[5] ^ st[10] ^ st[15] ^ st[20] ^ rotate(st[2] ^ st[7] ^ st[12] ^ st[17] ^ st[22], 1UL); + bc[1] = st[1] ^ st[6] ^ st[11] ^ st[16] ^ st[21] ^ rotate(st[3] ^ st[8] ^ st[13] ^ st[18] ^ st[23], 1UL); + bc[2] = st[2] ^ st[7] ^ st[12] ^ st[17] ^ st[22] ^ rotate(st[4] ^ st[9] ^ st[14] ^ st[19] ^ st[24], 1UL); + bc[3] = st[3] ^ st[8] ^ st[13] ^ st[18] ^ st[23] ^ rotate(st[0] ^ st[5] ^ st[10] ^ st[15] ^ st[20], 1UL); + bc[4] = st[4] ^ st[9] ^ st[14] ^ st[19] ^ st[24] ^ rotate(st[1] ^ st[6] ^ st[11] ^ st[16] ^ st[21], 1UL); + + st[0] ^= bc[4]; + st[5] ^= bc[4]; + st[10] ^= bc[4]; + st[15] ^= bc[4]; + st[20] ^= bc[4]; + + st[1] ^= bc[0]; + st[6] ^= bc[0]; + st[11] ^= bc[0]; + st[16] ^= bc[0]; + st[21] ^= bc[0]; + + st[2] ^= bc[1]; + st[7] ^= bc[1]; + st[12] ^= bc[1]; + st[17] ^= bc[1]; + st[22] ^= bc[1]; + + st[3] ^= bc[2]; + st[8] ^= bc[2]; + st[13] ^= bc[2]; + st[18] ^= bc[2]; + st[23] ^= bc[2]; + + st[4] ^= bc[3]; + st[9] ^= bc[3]; + st[14] ^= bc[3]; + st[19] ^= bc[3]; + st[24] ^= bc[3]; + + // Rho Pi + t = st[1]; + #pragma unroll + for (i = 0; i < 24; ++i) { + bc[0] = st[keccakf_piln[i]]; + st[keccakf_piln[i]] = rotate(t, (ulong)keccakf_rotc[i]); + t = bc[0]; + } + + #pragma unroll + for(int i = 0; i < 25; i += 5) + { + ulong tmp1 = st[i], tmp2 = st[i + 1]; + + st[i] = bitselect(st[i] ^ st[i + 2], st[i], st[i + 1]); + st[i + 1] = bitselect(st[i + 1] ^ st[i + 3], st[i + 1], st[i + 2]); + st[i + 2] = bitselect(st[i + 2] ^ st[i + 4], st[i + 2], st[i + 3]); + st[i + 3] = bitselect(st[i + 3] ^ tmp1, st[i + 3], st[i + 4]); + st[i + 4] = bitselect(st[i + 4] ^ tmp2, st[i + 4], tmp1); + } + + // Iota + st[0] ^= keccakf_rndc[round]; + } +} + +)===" +R"===( + +void CNKeccak(ulong *output, ulong *input) +{ + ulong st[25]; + + // Copy 72 bytes + for(int i = 0; i < 9; ++i) st[i] = input[i]; + + // Last four and '1' bit for padding + //st[9] = as_ulong((uint2)(((uint *)input)[18], 0x00000001U)); + + st[9] = (input[9] & 0x00000000FFFFFFFFUL) | 0x0000000100000000UL; + + for(int i = 10; i < 25; ++i) st[i] = 0x00UL; + + // Last bit of padding + st[16] = 0x8000000000000000UL; + + keccakf1600_1(st); + + for(int i = 0; i < 25; ++i) output[i] = st[i]; +} + +static const __constant uchar rcon[8] = { 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40 }; + +#define SubWord(inw) ((sbox[BYTE(inw, 3)] << 24) | (sbox[BYTE(inw, 2)] << 16) | (sbox[BYTE(inw, 1)] << 8) | sbox[BYTE(inw, 0)]) + +void AESExpandKey256(uint *keybuf) +{ + //#pragma unroll 4 + for(uint c = 8, i = 1; c < 40; ++c) + { + // For 256-bit keys, an sbox permutation is done every other 4th uint generated, AND every 8th + uint t = ((!(c & 7)) || ((c & 7) == 4)) ? SubWord(keybuf[c - 1]) : keybuf[c - 1]; + + // If the uint we're generating has an index that is a multiple of 8, rotate and XOR with the round constant, + // then XOR this with previously generated uint. If it's 4 after a multiple of 8, only the sbox permutation + // is done, followed by the XOR. If neither are true, only the XOR with the previously generated uint is done. + keybuf[c] = keybuf[c - 8] ^ ((!(c & 7)) ? rotate(t, 24U) ^ as_uint((uchar4)(rcon[i++], 0U, 0U, 0U)) : t); + } +} + +#define MEM_CHUNK (1 << MEM_CHUNK_EXPONENT) + +#if (STRIDED_INDEX == 0) +# define IDX(x) (x) +#elif (STRIDED_INDEX == 1) +# if (ALGO == CRYPTONIGHT_HEAVY) +# define IDX(x) ((x) * WORKSIZE) +# else +# define IDX(x) mul24((x), Threads) +# endif +#elif (STRIDED_INDEX == 2) +# define IDX(x) (((x) % MEM_CHUNK) + ((x) / MEM_CHUNK) * WORKSIZE * MEM_CHUNK) +#endif + +inline ulong getIdx() +{ +# if (STRIDED_INDEX == 0 || STRIDED_INDEX == 1 || STRIDED_INDEX == 2) + return get_global_id(0) - get_global_offset(0); +# endif +} + +//#include "opencl/cryptonight_gpu.cl" +XMRIG_INCLUDE_CN_GPU + +#define mix_and_propagate(xin) (xin)[(get_local_id(1)) % 8][get_local_id(0)] ^ (xin)[(get_local_id(1) + 1) % 8][get_local_id(0)] + +__attribute__((reqd_work_group_size(8, 8, 1))) +__kernel void cn0(__global ulong *input, __global uint4 *Scratchpad, __global ulong *states, uint Threads) +{ + uint ExpandedKey1[40]; + __local uint AES0[256], AES1[256], AES2[256], AES3[256]; + uint4 text; + + const uint gIdx = getIdx(); + + for (int i = get_local_id(1) * 8 + get_local_id(0); i < 256; i += 8 * 8) { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + AES2[i] = rotate(tmp, 16U); + AES3[i] = rotate(tmp, 24U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + + __local ulong State_buf[8 * 25]; + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; + +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) +# if (ALGO == CRYPTONIGHT_HEAVY) + Scratchpad += (gIdx / WORKSIZE) * (MEMORY >> 4) * WORKSIZE + (gIdx % WORKSIZE); +# else + Scratchpad += gIdx; +# endif +# elif (STRIDED_INDEX == 2) + Scratchpad += (gIdx / WORKSIZE) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * (gIdx % WORKSIZE); +# endif + + if (get_local_id(1) == 0) + { + __local ulong* State = State_buf + get_local_id(0) * 25; + + ((__local ulong8 *)State)[0] = vload8(0, input); + State[8] = input[8]; + State[9] = input[9]; + State[10] = input[10]; + State[11] = input[11]; + State[12] = input[12]; + State[13] = input[13]; + State[14] = input[14]; + State[15] = input[15]; + + ((__local uint *)State)[9] &= 0x00FFFFFFU; + ((__local uint *)State)[9] |= (((uint)get_global_id(0)) & 0xFF) << 24; + ((__local uint *)State)[10] &= 0xFF000000U; + /* explicit cast to `uint` is required because some OpenCL implementations (e.g. NVIDIA) + * handle get_global_id and get_global_offset as signed long long int and add + * 0xFFFFFFFF... to `get_global_id` if we set on host side a 32bit offset where the first bit is `1` + * (even if it is correct casted to unsigned on the host) + */ + ((__local uint *)State)[10] |= (((uint)get_global_id(0) >> 8)); + + // Last bit of padding + State[16] = 0x8000000000000000UL; + + for (int i = 17; i < 25; ++i) { + State[i] = 0x00UL; + } + + keccakf1600_2(State); + + #pragma unroll + for (int i = 0; i < 25; ++i) { + states[i] = State[i]; + } + } + } + + barrier(CLK_GLOBAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + text = vload4(get_local_id(1) + 4, (__global uint *)(states)); + + #pragma unroll + for (int i = 0; i < 4; ++i) { + ((ulong *)ExpandedKey1)[i] = states[i]; + } + + AESExpandKey256(ExpandedKey1); + } + + mem_fence(CLK_LOCAL_MEM_FENCE); + +# if (ALGO == CRYPTONIGHT_HEAVY) + { + __local uint4 xin[8][8]; + + /* Also left over threads perform this loop. + * The left over thread results will be ignored + */ + #pragma unroll 16 + for (size_t i = 0; i < 16; i++) { + #pragma unroll 10 + for (int j = 0; j < 10; ++j) { + uint4 t = ((uint4 *)ExpandedKey1)[j]; + t.s0 ^= AES0[BYTE(text.s0, 0)] ^ AES1[BYTE(text.s1, 1)] ^ AES2[BYTE(text.s2, 2)] ^ AES3[BYTE(text.s3, 3)]; + t.s1 ^= AES0[BYTE(text.s1, 0)] ^ AES1[BYTE(text.s2, 1)] ^ AES2[BYTE(text.s3, 2)] ^ AES3[BYTE(text.s0, 3)]; + t.s2 ^= AES0[BYTE(text.s2, 0)] ^ AES1[BYTE(text.s3, 1)] ^ AES2[BYTE(text.s0, 2)] ^ AES3[BYTE(text.s1, 3)]; + t.s3 ^= AES0[BYTE(text.s3, 0)] ^ AES1[BYTE(text.s0, 1)] ^ AES2[BYTE(text.s1, 2)] ^ AES3[BYTE(text.s2, 3)]; + text = t; + } + + barrier(CLK_LOCAL_MEM_FENCE); + xin[get_local_id(1)][get_local_id(0)] = text; + barrier(CLK_LOCAL_MEM_FENCE); + text = mix_and_propagate(xin); + } + } +# endif + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + const uint local_id1 = get_local_id(1); + #pragma unroll 2 + for(uint i = 0; i < (MEMORY >> 4); i += 8) { + #pragma unroll 10 + for (uint j = 0; j < 10; ++j) { + uint4 t = ((uint4 *)ExpandedKey1)[j]; + t.s0 ^= AES0[BYTE(text.s0, 0)] ^ AES1[BYTE(text.s1, 1)] ^ AES2[BYTE(text.s2, 2)] ^ AES3[BYTE(text.s3, 3)]; + t.s1 ^= AES0[BYTE(text.s1, 0)] ^ AES1[BYTE(text.s2, 1)] ^ AES2[BYTE(text.s3, 2)] ^ AES3[BYTE(text.s0, 3)]; + t.s2 ^= AES0[BYTE(text.s2, 0)] ^ AES1[BYTE(text.s3, 1)] ^ AES2[BYTE(text.s0, 2)] ^ AES3[BYTE(text.s1, 3)]; + t.s3 ^= AES0[BYTE(text.s3, 0)] ^ AES1[BYTE(text.s0, 1)] ^ AES2[BYTE(text.s1, 2)] ^ AES3[BYTE(text.s2, 3)]; + text = t; + } + + Scratchpad[IDX(i + local_id1)] = text; + } + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +} + +)===" +R"===( + +#define VARIANT1_1(p) \ + uint table = 0x75310U; \ + uint index = (((p).s2 >> 26) & 12) | (((p).s2 >> 23) & 2); \ + (p).s2 ^= ((table >> index) & 0x30U) << 24 + +#define VARIANT1_1_XTL(p) \ + uint table = 0x75310U; \ + uint offset = variant == VARIANT_XTL ? 27 : 26; \ + uint index = (((p).s2 >> offset) & 12) | (((p).s2 >> 23) & 2); \ + (p).s2 ^= ((table >> index) & 0x30U) << 24 + +#define VARIANT1_2(p) ((uint2 *)&(p))[0] ^= tweak1_2_0 + +#define VARIANT1_INIT() \ + tweak1_2 = as_uint2(input[4]); \ + tweak1_2.s0 >>= 24; \ + tweak1_2.s0 |= tweak1_2.s1 << 8; \ + tweak1_2.s1 = (uint) get_global_id(0); \ + tweak1_2 ^= as_uint2(states[24]) + +__attribute__((reqd_work_group_size(WORKSIZE, 1, 1))) +__kernel void cn1_monero(__global uint4 *Scratchpad, __global ulong *states, uint variant, __global ulong *input, uint Threads) +{ + ulong a[2], b[2]; + __local uint AES0[256], AES1[256]; + + const ulong gIdx = getIdx(); + + for (int i = get_local_id(0); i < 256; i += WORKSIZE) { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + + uint2 tweak1_2; + uint4 b_x; +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) +# if (ALGO == CRYPTONIGHT_HEAVY) + Scratchpad += (gIdx / WORKSIZE) * (MEMORY >> 4) * WORKSIZE + (gIdx % WORKSIZE); +# else + Scratchpad += gIdx; +# endif +# elif (STRIDED_INDEX == 2) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * get_local_id(0); +# endif + + a[0] = states[0] ^ states[4]; + b[0] = states[2] ^ states[6]; + a[1] = states[1] ^ states[5]; + b[1] = states[3] ^ states[7]; + + b_x = ((uint4 *)b)[0]; + VARIANT1_INIT(); + } + + mem_fence(CLK_LOCAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + #pragma unroll UNROLL_FACTOR + for (int i = 0; i < ITERATIONS; ++i) { + ulong c[2]; + + ((uint4 *)c)[0] = Scratchpad[IDX((as_uint2(a[0]).s0 & MASK) >> 4)]; + ((uint4 *)c)[0] = AES_Round_Two_Tables(AES0, AES1, ((uint4 *)c)[0], ((uint4 *)a)[0]); + + b_x ^= ((uint4 *)c)[0]; + VARIANT1_1_XTL(b_x); + Scratchpad[IDX((as_uint2(a[0]).s0 & MASK) >> 4)] = b_x; + + uint4 tmp; + tmp = Scratchpad[IDX((as_uint2(c[0]).s0 & MASK) >> 4)]; + + a[1] += c[0] * as_ulong2(tmp).s0; + a[0] += mul_hi(c[0], as_ulong2(tmp).s0); + + uint2 tweak1_2_0 = tweak1_2; + if (variant == VARIANT_RTO) { + tweak1_2_0 ^= ((uint2 *)&(a[0]))[0]; + } + + VARIANT1_2(a[1]); + Scratchpad[IDX((as_uint2(c[0]).s0 & MASK) >> 4)] = ((uint4 *)a)[0]; + VARIANT1_2(a[1]); + + ((uint4 *)a)[0] ^= tmp; + + b_x = ((uint4 *)c)[0]; + } + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +} + + +)===" +R"===( + +__attribute__((reqd_work_group_size(WORKSIZE, 1, 1))) +__kernel void cn1_v2_monero(__global uint4 *Scratchpad, __global ulong *states, uint variant, __global ulong *input, uint Threads) +{ +# if (ALGO == CRYPTONIGHT || ALGO == CRYPTONIGHT_PICO) + ulong a[2], b[4]; + __local uint AES0[256], AES1[256], AES2[256], AES3[256]; + + const ulong gIdx = getIdx(); + + for(int i = get_local_id(0); i < 256; i += WORKSIZE) + { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + AES2[i] = rotate(tmp, 16U); + AES3[i] = rotate(tmp, 24U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; + +# if defined(__NV_CL_C_VERSION) + Scratchpad += gIdx * (ITERATIONS >> 2); +# else +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) + Scratchpad += gIdx; +# elif (STRIDED_INDEX == 2) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * get_local_id(0); +# endif +# endif + + a[0] = states[0] ^ states[4]; + a[1] = states[1] ^ states[5]; + + b[0] = states[2] ^ states[6]; + b[1] = states[3] ^ states[7]; + b[2] = states[8] ^ states[10]; + b[3] = states[9] ^ states[11]; + } + + ulong2 bx0 = ((ulong2 *)b)[0]; + ulong2 bx1 = ((ulong2 *)b)[1]; + + mem_fence(CLK_LOCAL_MEM_FENCE); + +# ifdef __NV_CL_C_VERSION + __local uint16 scratchpad_line_buf[WORKSIZE]; + __local uint16* scratchpad_line = scratchpad_line_buf + get_local_id(0); +# define SCRATCHPAD_CHUNK(N) (*(__local uint4*)((__local uchar*)(scratchpad_line) + (idx1 ^ (N << 4)))) +# else +# if (STRIDED_INDEX == 0) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + (idx ^ (N << 4)))) +# elif (STRIDED_INDEX == 1) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + mul24(as_uint(idx ^ (N << 4)), Threads))) +# elif (STRIDED_INDEX == 2) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + (((idx ^ (N << 4)) % (MEM_CHUNK << 4)) + ((idx ^ (N << 4)) / (MEM_CHUNK << 4)) * WORKSIZE * (MEM_CHUNK << 4)))) +# endif +# endif + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + uint2 division_result = as_uint2(states[12]); + uint sqrt_result = as_uint2(states[13]).s0; + + #pragma unroll UNROLL_FACTOR + for(int i = 0; i < ITERATIONS; ++i) + { +# ifdef __NV_CL_C_VERSION + uint idx = a[0] & 0x1FFFC0; + uint idx1 = a[0] & 0x30; + + *scratchpad_line = *(__global uint16*)((__global uchar*)(Scratchpad) + idx); +# else + uint idx = a[0] & MASK; +# endif + + uint4 c = SCRATCHPAD_CHUNK(0); + c = AES_Round(AES0, AES1, AES2, AES3, c, ((uint4 *)a)[0]); + + { + const ulong2 chunk1 = as_ulong2(SCRATCHPAD_CHUNK(1)); + const ulong2 chunk2 = as_ulong2(SCRATCHPAD_CHUNK(2)); + const ulong2 chunk3 = as_ulong2(SCRATCHPAD_CHUNK(3)); + + SCRATCHPAD_CHUNK(1) = as_uint4(chunk3 + bx1); + SCRATCHPAD_CHUNK(2) = as_uint4(chunk1 + bx0); + SCRATCHPAD_CHUNK(3) = as_uint4(chunk2 + ((ulong2 *)a)[0]); + } + + SCRATCHPAD_CHUNK(0) = as_uint4(bx0) ^ c; + +# ifdef __NV_CL_C_VERSION + *(__global uint16*)((__global uchar*)(Scratchpad) + idx) = *scratchpad_line; + + idx = as_ulong2(c).s0 & 0x1FFFC0; + idx1 = as_ulong2(c).s0 & 0x30; + + *scratchpad_line = *(__global uint16*)((__global uchar*)(Scratchpad) + idx); +# else + idx = as_ulong2(c).s0 & MASK; +# endif + + uint4 tmp = SCRATCHPAD_CHUNK(0); + + { + tmp.s0 ^= division_result.s0; + tmp.s1 ^= division_result.s1 ^ sqrt_result; + + division_result = fast_div_v2(as_ulong2(c).s1, (c.s0 + (sqrt_result << 1)) | 0x80000001UL); + sqrt_result = fast_sqrt_v2(as_ulong2(c).s0 + as_ulong(division_result)); + } + + ulong2 t; + t.s0 = mul_hi(as_ulong2(c).s0, as_ulong2(tmp).s0); + t.s1 = as_ulong2(c).s0 * as_ulong2(tmp).s0; + { + const ulong2 chunk1 = as_ulong2(SCRATCHPAD_CHUNK(1)) ^ t; + const ulong2 chunk2 = as_ulong2(SCRATCHPAD_CHUNK(2)); + t ^= chunk2; + const ulong2 chunk3 = as_ulong2(SCRATCHPAD_CHUNK(3)); + + SCRATCHPAD_CHUNK(1) = as_uint4(chunk3 + bx1); + SCRATCHPAD_CHUNK(2) = as_uint4(chunk1 + bx0); + SCRATCHPAD_CHUNK(3) = as_uint4(chunk2 + ((ulong2 *)a)[0]); + } + + a[1] += t.s1; + a[0] += t.s0; + + SCRATCHPAD_CHUNK(0) = ((uint4 *)a)[0]; + +# ifdef __NV_CL_C_VERSION + *(__global uint16*)((__global uchar*)(Scratchpad) + idx) = *scratchpad_line; +# endif + + ((uint4 *)a)[0] ^= tmp; + bx1 = bx0; + bx0 = as_ulong2(c); + } + +# undef SCRATCHPAD_CHUNK + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +# endif +} + +)===" +R"===( + +__attribute__((reqd_work_group_size(WORKSIZE, 1, 1))) +__kernel void cn1_v2_half(__global uint4 *Scratchpad, __global ulong *states, uint variant, __global ulong *input, uint Threads) +{ +# if (ALGO == CRYPTONIGHT) + ulong a[2], b[4]; + __local uint AES0[256], AES1[256], AES2[256], AES3[256]; + + const ulong gIdx = getIdx(); + + for(int i = get_local_id(0); i < 256; i += WORKSIZE) + { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + AES2[i] = rotate(tmp, 16U); + AES3[i] = rotate(tmp, 24U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; + +# if defined(__NV_CL_C_VERSION) + Scratchpad += gIdx * (0x40000 >> 2); +# else +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) + Scratchpad += gIdx; +# elif (STRIDED_INDEX == 2) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * get_local_id(0); +# endif +# endif + + a[0] = states[0] ^ states[4]; + a[1] = states[1] ^ states[5]; + + b[0] = states[2] ^ states[6]; + b[1] = states[3] ^ states[7]; + b[2] = states[8] ^ states[10]; + b[3] = states[9] ^ states[11]; + } + + ulong2 bx0 = ((ulong2 *)b)[0]; + ulong2 bx1 = ((ulong2 *)b)[1]; + + mem_fence(CLK_LOCAL_MEM_FENCE); + +# ifdef __NV_CL_C_VERSION + __local uint16 scratchpad_line_buf[WORKSIZE]; + __local uint16* scratchpad_line = scratchpad_line_buf + get_local_id(0); +# define SCRATCHPAD_CHUNK(N) (*(__local uint4*)((__local uchar*)(scratchpad_line) + (idx1 ^ (N << 4)))) +# else +# if (STRIDED_INDEX == 0) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + (idx ^ (N << 4)))) +# elif (STRIDED_INDEX == 1) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + mul24(as_uint(idx ^ (N << 4)), Threads))) +# elif (STRIDED_INDEX == 2) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + (((idx ^ (N << 4)) % (MEM_CHUNK << 4)) + ((idx ^ (N << 4)) / (MEM_CHUNK << 4)) * WORKSIZE * (MEM_CHUNK << 4)))) +# endif +# endif + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + uint2 division_result = as_uint2(states[12]); + uint sqrt_result = as_uint2(states[13]).s0; + + #pragma unroll UNROLL_FACTOR + for(int i = 0; i < 0x40000; ++i) + { +# ifdef __NV_CL_C_VERSION + uint idx = a[0] & 0x1FFFC0; + uint idx1 = a[0] & 0x30; + + *scratchpad_line = *(__global uint16*)((__global uchar*)(Scratchpad) + idx); +# else + uint idx = a[0] & MASK; +# endif + + uint4 c = SCRATCHPAD_CHUNK(0); + c = AES_Round(AES0, AES1, AES2, AES3, c, ((uint4 *)a)[0]); + + { + const ulong2 chunk1 = as_ulong2(SCRATCHPAD_CHUNK(1)); + const ulong2 chunk2 = as_ulong2(SCRATCHPAD_CHUNK(2)); + const ulong2 chunk3 = as_ulong2(SCRATCHPAD_CHUNK(3)); + + SCRATCHPAD_CHUNK(1) = as_uint4(chunk3 + bx1); + SCRATCHPAD_CHUNK(2) = as_uint4(chunk1 + bx0); + SCRATCHPAD_CHUNK(3) = as_uint4(chunk2 + ((ulong2 *)a)[0]); + } + + SCRATCHPAD_CHUNK(0) = as_uint4(bx0) ^ c; + +# ifdef __NV_CL_C_VERSION + *(__global uint16*)((__global uchar*)(Scratchpad) + idx) = *scratchpad_line; + + idx = as_ulong2(c).s0 & 0x1FFFC0; + idx1 = as_ulong2(c).s0 & 0x30; + + *scratchpad_line = *(__global uint16*)((__global uchar*)(Scratchpad) + idx); +# else + idx = as_ulong2(c).s0 & MASK; +# endif + + uint4 tmp = SCRATCHPAD_CHUNK(0); + + { + tmp.s0 ^= division_result.s0; + tmp.s1 ^= division_result.s1 ^ sqrt_result; + + division_result = fast_div_v2(as_ulong2(c).s1, (c.s0 + (sqrt_result << 1)) | 0x80000001UL); + sqrt_result = fast_sqrt_v2(as_ulong2(c).s0 + as_ulong(division_result)); + } + + ulong2 t; + t.s0 = mul_hi(as_ulong2(c).s0, as_ulong2(tmp).s0); + t.s1 = as_ulong2(c).s0 * as_ulong2(tmp).s0; + { + const ulong2 chunk1 = as_ulong2(SCRATCHPAD_CHUNK(1)) ^ t; + const ulong2 chunk2 = as_ulong2(SCRATCHPAD_CHUNK(2)); + t ^= chunk2; + const ulong2 chunk3 = as_ulong2(SCRATCHPAD_CHUNK(3)); + + SCRATCHPAD_CHUNK(1) = as_uint4(chunk3 + bx1); + SCRATCHPAD_CHUNK(2) = as_uint4(chunk1 + bx0); + SCRATCHPAD_CHUNK(3) = as_uint4(chunk2 + ((ulong2 *)a)[0]); + } + + a[1] += t.s1; + a[0] += t.s0; + + SCRATCHPAD_CHUNK(0) = ((uint4 *)a)[0]; + +# ifdef __NV_CL_C_VERSION + *(__global uint16*)((__global uchar*)(Scratchpad) + idx) = *scratchpad_line; +# endif + + ((uint4 *)a)[0] ^= tmp; + bx1 = bx0; + bx0 = as_ulong2(c); + } + +# undef SCRATCHPAD_CHUNK + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +# endif +} + +)===" +R"===( + +__attribute__((reqd_work_group_size(WORKSIZE, 1, 1))) +__kernel void cn1_msr(__global uint4 *Scratchpad, __global ulong *states, uint variant, __global ulong *input, uint Threads) +{ +# if (ALGO == CRYPTONIGHT) + ulong a[2], b[2]; + __local uint AES0[256], AES1[256]; + + const ulong gIdx = getIdx(); + + for (int i = get_local_id(0); i < 256; i += WORKSIZE) { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + + uint2 tweak1_2; + uint4 b_x; +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) +# if (ALGO == CRYPTONIGHT_HEAVY) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + get_local_id(0); +# else + Scratchpad += gIdx; +# endif +# elif (STRIDED_INDEX == 2) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * get_local_id(0); +# endif + + a[0] = states[0] ^ states[4]; + b[0] = states[2] ^ states[6]; + a[1] = states[1] ^ states[5]; + b[1] = states[3] ^ states[7]; + + b_x = ((uint4 *)b)[0]; + VARIANT1_INIT(); + } + + mem_fence(CLK_LOCAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + #pragma unroll 8 + for (int i = 0; i < 0x40000; ++i) { + ulong c[2]; + + ((uint4 *)c)[0] = Scratchpad[IDX((as_uint2(a[0]).s0 & MASK) >> 4)]; + ((uint4 *)c)[0] = AES_Round_Two_Tables(AES0, AES1, ((uint4 *)c)[0], ((uint4 *)a)[0]); + + b_x ^= ((uint4 *)c)[0]; + VARIANT1_1(b_x); + Scratchpad[IDX((as_uint2(a[0]).s0 & MASK) >> 4)] = b_x; + + uint4 tmp; + tmp = Scratchpad[IDX((as_uint2(c[0]).s0 & MASK) >> 4)]; + + a[1] += c[0] * as_ulong2(tmp).s0; + a[0] += mul_hi(c[0], as_ulong2(tmp).s0); + + uint2 tweak1_2_0 = tweak1_2; + + VARIANT1_2(a[1]); + Scratchpad[IDX((as_uint2(c[0]).s0 & MASK) >> 4)] = ((uint4 *)a)[0]; + VARIANT1_2(a[1]); + + ((uint4 *)a)[0] ^= tmp; + + b_x = ((uint4 *)c)[0]; + } + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +# endif +} + +)===" +R"===( + +__attribute__((reqd_work_group_size(WORKSIZE, 1, 1))) +__kernel void cn1_tube(__global uint4 *Scratchpad, __global ulong *states, uint variant, __global ulong *input, uint Threads) +{ +# if (ALGO == CRYPTONIGHT_HEAVY) + ulong a[2], b[2]; + __local uint AES0[256], AES1[256]; + + const ulong gIdx = getIdx(); + + for (int i = get_local_id(0); i < 256; i += WORKSIZE) { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + + uint2 tweak1_2; + uint4 b_x; +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) +# if (ALGO == CRYPTONIGHT_HEAVY) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + get_local_id(0); +# else + Scratchpad += gIdx; +# endif +# elif (STRIDED_INDEX == 2) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * get_local_id(0); +# endif + + a[0] = states[0] ^ states[4]; + b[0] = states[2] ^ states[6]; + a[1] = states[1] ^ states[5]; + b[1] = states[3] ^ states[7]; + + b_x = ((uint4 *)b)[0]; + VARIANT1_INIT(); + } + + mem_fence(CLK_LOCAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + uint idx0 = a[0]; + + #pragma unroll UNROLL_FACTOR + for (int i = 0; i < ITERATIONS; ++i) { + ulong c[2]; + + ((uint4 *)c)[0] = Scratchpad[IDX((idx0 & MASK) >> 4)]; + ((uint4 *)c)[0] = AES_Round_bittube2(AES0, AES1, ((uint4 *)c)[0], ((uint4 *)a)[0]); + + b_x ^= ((uint4 *)c)[0]; + VARIANT1_1(b_x); + Scratchpad[IDX((idx0 & MASK) >> 4)] = b_x; + + uint4 tmp; + tmp = Scratchpad[IDX((as_uint2(c[0]).s0 & MASK) >> 4)]; + + a[1] += c[0] * as_ulong2(tmp).s0; + a[0] += mul_hi(c[0], as_ulong2(tmp).s0); + + uint2 tweak1_2_0 = tweak1_2; + tweak1_2_0 ^= ((uint2 *)&(a[0]))[0]; + + VARIANT1_2(a[1]); + Scratchpad[IDX((as_uint2(c[0]).s0 & MASK) >> 4)] = ((uint4 *)a)[0]; + VARIANT1_2(a[1]); + + ((uint4 *)a)[0] ^= tmp; + idx0 = a[0]; + + b_x = ((uint4 *)c)[0]; + + { + long n = *((__global long*)(Scratchpad + (IDX((idx0 & MASK) >> 4)))); + int d = ((__global int*)(Scratchpad + (IDX((idx0 & MASK) >> 4))))[2]; + long q = fast_div_heavy(n, d | 0x5); + *((__global long*)(Scratchpad + (IDX((idx0 & MASK) >> 4)))) = n ^ q; + idx0 = d ^ q; + } + } + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +# endif +} + +)===" +R"===( + +__attribute__((reqd_work_group_size(WORKSIZE, 1, 1))) +__kernel void cn1(__global uint4 *Scratchpad, __global ulong *states, uint variant, __global ulong *input, uint Threads) +{ + ulong a[2], b[2]; + __local uint AES0[256], AES1[256]; + + const ulong gIdx = getIdx(); + + for (int i = get_local_id(0); i < 256; i += WORKSIZE) { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + + uint4 b_x; +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) +# if (ALGO == CRYPTONIGHT_HEAVY) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + get_local_id(0); +# else + Scratchpad += gIdx; +# endif +# elif(STRIDED_INDEX == 2) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * get_local_id(0); +# endif + + a[0] = states[0] ^ states[4]; + b[0] = states[2] ^ states[6]; + a[1] = states[1] ^ states[5]; + b[1] = states[3] ^ states[7]; + + b_x = ((uint4 *)b)[0]; + } + + mem_fence(CLK_LOCAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + uint idx0 = a[0]; + + #pragma unroll UNROLL_FACTOR + for (int i = 0; i < ITERATIONS; ++i) { + ulong c[2]; + + ((uint4 *)c)[0] = Scratchpad[IDX((idx0 & MASK) >> 4)]; + ((uint4 *)c)[0] = AES_Round_Two_Tables(AES0, AES1, ((uint4 *)c)[0], ((uint4 *)a)[0]); + + Scratchpad[IDX((idx0 & MASK) >> 4)] = b_x ^ ((uint4 *)c)[0]; + + uint4 tmp; + tmp = Scratchpad[IDX((as_uint2(c[0]).s0 & MASK) >> 4)]; + + a[1] += c[0] * as_ulong2(tmp).s0; + a[0] += mul_hi(c[0], as_ulong2(tmp).s0); + + Scratchpad[IDX((as_uint2(c[0]).s0 & MASK) >> 4)] = ((uint4 *)a)[0]; + + ((uint4 *)a)[0] ^= tmp; + idx0 = a[0]; + + b_x = ((uint4 *)c)[0]; + +# if (ALGO == CRYPTONIGHT_HEAVY) + { + const long2 n = *((__global long2*)(Scratchpad + (IDX((idx0 & MASK) >> 4)))); + long q = fast_div_heavy(n.s0, as_int4(n).s2 | 0x5); + *((__global long*)(Scratchpad + (IDX((idx0 & MASK) >> 4)))) = n.s0 ^ q; + + if (variant == VARIANT_XHV) { + idx0 = (~as_int4(n).s2) ^ q; + } else { + idx0 = as_int4(n).s2 ^ q; + } + } +# endif + } + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +} + +)===" +R"===( + +__attribute__((reqd_work_group_size(WORKSIZE, 1, 1))) +__kernel void cn1_xao(__global uint4 *Scratchpad, __global ulong *states, uint variant, __global ulong *input, uint Threads) +{ +# if (ALGO == CRYPTONIGHT) + ulong a[2], b[2]; + __local uint AES0[256], AES1[256]; + + const ulong gIdx = getIdx(); + + for (int i = get_local_id(0); i < 256; i += WORKSIZE) { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + + uint4 b_x; +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) +# if (ALGO == CRYPTONIGHT_HEAVY) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + get_local_id(0); +# else + Scratchpad += gIdx; +# endif +# elif(STRIDED_INDEX == 2) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * get_local_id(0); +# endif + + a[0] = states[0] ^ states[4]; + b[0] = states[2] ^ states[6]; + a[1] = states[1] ^ states[5]; + b[1] = states[3] ^ states[7]; + + b_x = ((uint4 *)b)[0]; + } + + mem_fence(CLK_LOCAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + uint idx0 = a[0]; + + #pragma unroll 8 + for (int i = 0; i < 0x100000; ++i) { + ulong c[2]; + + ((uint4 *)c)[0] = Scratchpad[IDX((idx0 & MASK) >> 4)]; + ((uint4 *)c)[0] = AES_Round_Two_Tables(AES0, AES1, ((uint4 *)c)[0], ((uint4 *)a)[0]); + + Scratchpad[IDX((idx0 & MASK) >> 4)] = b_x ^ ((uint4 *)c)[0]; + + uint4 tmp; + tmp = Scratchpad[IDX((as_uint2(c[0]).s0 & MASK) >> 4)]; + + a[1] += c[0] * as_ulong2(tmp).s0; + a[0] += mul_hi(c[0], as_ulong2(tmp).s0); + + Scratchpad[IDX((as_uint2(c[0]).s0 & MASK) >> 4)] = ((uint4 *)a)[0]; + + ((uint4 *)a)[0] ^= tmp; + idx0 = a[0]; + + b_x = ((uint4 *)c)[0]; + } + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +# endif +} + +)===" +R"===( + +__attribute__((reqd_work_group_size(8, 8, 1))) +__kernel void cn2(__global uint4 *Scratchpad, __global ulong *states, __global uint *Branch0, __global uint *Branch1, __global uint *Branch2, __global uint *Branch3, uint Threads) +{ + __local uint AES0[256], AES1[256], AES2[256], AES3[256]; + uint ExpandedKey2[40]; + uint4 text; + + const ulong gIdx = getIdx(); + + for (int i = get_local_id(1) * 8 + get_local_id(0); i < 256; i += 8 * 8) { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + AES2[i] = rotate(tmp, 16U); + AES3[i] = rotate(tmp, 24U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) +# if (ALGO == CRYPTONIGHT_HEAVY) + Scratchpad += (gIdx / WORKSIZE) * (MEMORY >> 4) * WORKSIZE + (gIdx % WORKSIZE); +# else + Scratchpad += gIdx; +# endif +# elif (STRIDED_INDEX == 2) + Scratchpad += (gIdx / WORKSIZE) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * (gIdx % WORKSIZE); +# endif + + #if defined(__Tahiti__) || defined(__Pitcairn__) + + for(int i = 0; i < 4; ++i) ((ulong *)ExpandedKey2)[i] = states[i + 4]; + text = vload4(get_local_id(1) + 4, (__global uint *)states); + + #else + + text = vload4(get_local_id(1) + 4, (__global uint *)states); + ((uint8 *)ExpandedKey2)[0] = vload8(1, (__global uint *)states); + + #endif + + AESExpandKey256(ExpandedKey2); + } + + barrier(CLK_LOCAL_MEM_FENCE); + +# if (ALGO == CRYPTONIGHT_HEAVY) + __local uint4 xin1[8][8]; + __local uint4 xin2[8][8]; + __local uint4* xin1_store = &xin1[get_local_id(1)][get_local_id(0)]; + __local uint4* xin1_load = &xin1[(get_local_id(1) + 1) % 8][get_local_id(0)]; + __local uint4* xin2_store = &xin2[get_local_id(1)][get_local_id(0)]; + __local uint4* xin2_load = &xin2[(get_local_id(1) + 1) % 8][get_local_id(0)]; + *xin2_store = (uint4)(0, 0, 0, 0); +# endif + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { +# if (ALGO == CRYPTONIGHT_HEAVY) + #pragma unroll 2 + for(int i = 0, i1 = get_local_id(1); i < (MEMORY >> 7); ++i, i1 = (i1 + 16) % (MEMORY >> 4)) + { + text ^= Scratchpad[IDX(i1)]; + barrier(CLK_LOCAL_MEM_FENCE); + text ^= *xin2_load; + + #pragma unroll 10 + for(int j = 0; j < 10; ++j) + text = AES_Round(AES0, AES1, AES2, AES3, text, ((uint4 *)ExpandedKey2)[j]); + + *xin1_store = text; + + text ^= Scratchpad[IDX(i1 + 8)]; + barrier(CLK_LOCAL_MEM_FENCE); + text ^= *xin1_load; + + #pragma unroll 10 + for(int j = 0; j < 10; ++j) + text = AES_Round(AES0, AES1, AES2, AES3, text, ((uint4 *)ExpandedKey2)[j]); + + *xin2_store = text; + } + + barrier(CLK_LOCAL_MEM_FENCE); + text ^= *xin2_load; + +# else + const uint local_id1 = get_local_id(1); + #pragma unroll 2 + for (uint i = 0; i < (MEMORY >> 7); ++i) { + text ^= Scratchpad[IDX((i << 3) + local_id1)]; + + #pragma unroll 10 + for(uint j = 0; j < 10; ++j) + text = AES_Round(AES0, AES1, AES2, AES3, text, ((uint4 *)ExpandedKey2)[j]); + } +# endif + } + +# if (ALGO == CRYPTONIGHT_HEAVY) + /* Also left over threads performe this loop. + * The left over thread results will be ignored + */ + #pragma unroll 16 + for(size_t i = 0; i < 16; i++) + { + #pragma unroll 10 + for (int j = 0; j < 10; ++j) { + text = AES_Round(AES0, AES1, AES2, AES3, text, ((uint4 *)ExpandedKey2)[j]); + } + + barrier(CLK_LOCAL_MEM_FENCE); + *xin1_store = text; + barrier(CLK_LOCAL_MEM_FENCE); + text ^= *xin1_load; + } +# endif + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + vstore2(as_ulong2(text), get_local_id(1) + 4, states); + } + + barrier(CLK_GLOBAL_MEM_FENCE); + + __local ulong State_buf[8 * 25]; + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + if(!get_local_id(1)) + { + __local ulong* State = State_buf + get_local_id(0) * 25; + + for(int i = 0; i < 25; ++i) State[i] = states[i]; + + keccakf1600_2(State); + + for(int i = 0; i < 25; ++i) states[i] = State[i]; + + uint StateSwitch = State[0] & 3; + __global uint *destinationBranch1 = StateSwitch == 0 ? Branch0 : Branch1; + __global uint *destinationBranch2 = StateSwitch == 2 ? Branch2 : Branch3; + __global uint *destinationBranch = StateSwitch < 2 ? destinationBranch1 : destinationBranch2; + destinationBranch[atomic_inc(destinationBranch + Threads)] = gIdx; + } + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +} + +)===" +R"===( + +#define VSWAP8(x) (((x) >> 56) | (((x) >> 40) & 0x000000000000FF00UL) | (((x) >> 24) & 0x0000000000FF0000UL) \ + | (((x) >> 8) & 0x00000000FF000000UL) | (((x) << 8) & 0x000000FF00000000UL) \ + | (((x) << 24) & 0x0000FF0000000000UL) | (((x) << 40) & 0x00FF000000000000UL) | (((x) << 56) & 0xFF00000000000000UL)) + +#define VSWAP4(x) ((((x) >> 24) & 0xFFU) | (((x) >> 8) & 0xFF00U) | (((x) << 8) & 0xFF0000U) | (((x) << 24) & 0xFF000000U)) + +__kernel void Skein(__global ulong *states, __global uint *BranchBuf, __global uint *output, ulong Target, uint Threads) +{ + const uint idx = get_global_id(0) - get_global_offset(0); + + // do not use early return here + if(idx < Threads) + { + states += 25 * BranchBuf[idx]; + + // skein + ulong8 h = vload8(0, SKEIN512_256_IV); + + // Type field begins with final bit, first bit, then six bits of type; the last 96 + // bits are input processed (including in the block to be processed with that tweak) + // The output transform is only one run of UBI, since we need only 256 bits of output + // The tweak for the output transform is Type = Output with the Final bit set + // T[0] for the output is 8, and I don't know why - should be message size... + ulong t[3] = { 0x00UL, 0x7000000000000000UL, 0x00UL }; + ulong8 p, m; + + #pragma unroll 1 + for (uint i = 0; i < 4; ++i) + { + t[0] += i < 3 ? 0x40UL : 0x08UL; + + t[2] = t[0] ^ t[1]; + + m = (i < 3) ? vload8(i, states) : (ulong8)(states[24], 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL); + const ulong h8 = h.s0 ^ h.s1 ^ h.s2 ^ h.s3 ^ h.s4 ^ h.s5 ^ h.s6 ^ h.s7 ^ SKEIN_KS_PARITY; + p = Skein512Block(m, h, h8, t); + + h = m ^ p; + + t[1] = i < 2 ? 0x3000000000000000UL : 0xB000000000000000UL; + } + + t[0] = 0x08UL; + t[1] = 0xFF00000000000000UL; + t[2] = t[0] ^ t[1]; + + p = (ulong8)(0); + const ulong h8 = h.s0 ^ h.s1 ^ h.s2 ^ h.s3 ^ h.s4 ^ h.s5 ^ h.s6 ^ h.s7 ^ SKEIN_KS_PARITY; + + p = Skein512Block(p, h, h8, t); + + // Note that comparison is equivalent to subtraction - we can't just compare 8 32-bit values + // and expect an accurate result for target > 32-bit without implementing carries + if (p.s3 <= Target) { + ulong outIdx = atomic_inc(output + 0xFF); + if (outIdx < 0xFF) { + output[outIdx] = BranchBuf[idx] + (uint) get_global_offset(0); + } + } + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +} + +#define SWAP8(x) as_ulong(as_uchar8(x).s76543210) + +#define JHXOR \ + h0h ^= input[0]; \ + h0l ^= input[1]; \ + h1h ^= input[2]; \ + h1l ^= input[3]; \ + h2h ^= input[4]; \ + h2l ^= input[5]; \ + h3h ^= input[6]; \ + h3l ^= input[7]; \ +\ + E8; \ +\ + h4h ^= input[0]; \ + h4l ^= input[1]; \ + h5h ^= input[2]; \ + h5l ^= input[3]; \ + h6h ^= input[4]; \ + h6l ^= input[5]; \ + h7h ^= input[6]; \ + h7l ^= input[7] + +__kernel void JH(__global ulong *states, __global uint *BranchBuf, __global uint *output, ulong Target, uint Threads) +{ + const uint idx = get_global_id(0) - get_global_offset(0); + + // do not use early return here + if(idx < Threads) + { + states += 25 * BranchBuf[idx]; + + sph_u64 h0h = 0xEBD3202C41A398EBUL, h0l = 0xC145B29C7BBECD92UL, h1h = 0xFAC7D4609151931CUL, h1l = 0x038A507ED6820026UL, h2h = 0x45B92677269E23A4UL, h2l = 0x77941AD4481AFBE0UL, h3h = 0x7A176B0226ABB5CDUL, h3l = 0xA82FFF0F4224F056UL; + sph_u64 h4h = 0x754D2E7F8996A371UL, h4l = 0x62E27DF70849141DUL, h5h = 0x948F2476F7957627UL, h5l = 0x6C29804757B6D587UL, h6h = 0x6C0D8EAC2D275E5CUL, h6l = 0x0F7A0557C6508451UL, h7h = 0xEA12247067D3E47BUL, h7l = 0x69D71CD313ABE389UL; + sph_u64 tmp; + + for (uint i = 0; i < 3; ++i) { + ulong input[8]; + + const int shifted = i << 3; + for (uint x = 0; x < 8; ++x) { + input[x] = (states[shifted + x]); + } + + JHXOR; + } + + { + ulong input[8] = { (states[24]), 0x80UL, 0x00UL, 0x00UL, 0x00UL, 0x00UL, 0x00UL, 0x00UL }; + JHXOR; + } + + { + ulong input[8] = { 0x00UL, 0x00UL, 0x00UL, 0x00UL, 0x00UL, 0x00UL, 0x00UL, 0x4006000000000000UL }; + JHXOR; + } + + // Note that comparison is equivalent to subtraction - we can't just compare 8 32-bit values + // and expect an accurate result for target > 32-bit without implementing carries + if (h7l <= Target) { + ulong outIdx = atomic_inc(output + 0xFF); + if (outIdx < 0xFF) { + output[outIdx] = BranchBuf[idx] + (uint) get_global_offset(0); + } + } + } +} + +#define SWAP4(x) as_uint(as_uchar4(x).s3210) + +__kernel void Blake(__global ulong *states, __global uint *BranchBuf, __global uint *output, ulong Target, uint Threads) +{ + const uint idx = get_global_id(0) - get_global_offset(0); + + // do not use early return here + if (idx < Threads) + { + states += 25 * BranchBuf[idx]; + + unsigned int m[16]; + unsigned int v[16]; + uint h[8]; + uint bitlen = 0; + + ((uint8 *)h)[0] = vload8(0U, c_IV256); + + for (uint i = 0; i < 3; ++i) { + ((uint16 *)m)[0] = vload16(i, (__global uint *)states); + for (uint x = 0; x < 16; ++x) { + m[x] = SWAP4(m[x]); + } + + bitlen += 512; + + ((uint16 *)v)[0].lo = ((uint8 *)h)[0]; + ((uint16 *)v)[0].hi = vload8(0U, c_u256); + + v[12] ^= bitlen; + v[13] ^= bitlen; + + for (uint r = 0; r < 14; r++) { + GS(0, 4, 0x8, 0xC, 0x0); + GS(1, 5, 0x9, 0xD, 0x2); + GS(2, 6, 0xA, 0xE, 0x4); + GS(3, 7, 0xB, 0xF, 0x6); + GS(0, 5, 0xA, 0xF, 0x8); + GS(1, 6, 0xB, 0xC, 0xA); + GS(2, 7, 0x8, 0xD, 0xC); + GS(3, 4, 0x9, 0xE, 0xE); + } + + ((uint8 *)h)[0] ^= ((uint8 *)v)[0] ^ ((uint8 *)v)[1]; + } + + m[0] = SWAP4(((__global uint *)states)[48]); + m[1] = SWAP4(((__global uint *)states)[49]); + m[2] = 0x80000000U; + m[3] = 0x00U; + m[4] = 0x00U; + m[5] = 0x00U; + m[6] = 0x00U; + m[7] = 0x00U; + m[8] = 0x00U; + m[9] = 0x00U; + m[10] = 0x00U; + m[11] = 0x00U; + m[12] = 0x00U; + m[13] = 1U; + m[14] = 0U; + m[15] = 0x640; + + bitlen += 64; + + ((uint16 *)v)[0].lo = ((uint8 *)h)[0]; + ((uint16 *)v)[0].hi = vload8(0U, c_u256); + + v[12] ^= bitlen; + v[13] ^= bitlen; + + for (uint r = 0; r < 14; r++) { + GS(0, 4, 0x8, 0xC, 0x0); + GS(1, 5, 0x9, 0xD, 0x2); + GS(2, 6, 0xA, 0xE, 0x4); + GS(3, 7, 0xB, 0xF, 0x6); + GS(0, 5, 0xA, 0xF, 0x8); + GS(1, 6, 0xB, 0xC, 0xA); + GS(2, 7, 0x8, 0xD, 0xC); + GS(3, 4, 0x9, 0xE, 0xE); + } + + ((uint8 *)h)[0] ^= ((uint8 *)v)[0] ^ ((uint8 *)v)[1]; + + for (uint i = 0; i < 8; ++i) { + h[i] = SWAP4(h[i]); + } + + // Note that comparison is equivalent to subtraction - we can't just compare 8 32-bit values + // and expect an accurate result for target > 32-bit without implementing carries + uint2 t = (uint2)(h[6],h[7]); + if (as_ulong(t) <= Target) { + ulong outIdx = atomic_inc(output + 0xFF); + if (outIdx < 0xFF) { + output[outIdx] = BranchBuf[idx] + (uint) get_global_offset(0); + } + } + } +} + +#undef SWAP4 + + +__kernel void Groestl(__global ulong *states, __global uint *BranchBuf, __global uint *output, ulong Target, uint Threads) +{ + const uint idx = get_global_id(0) - get_global_offset(0); + + // do not use early return here + if (idx < Threads) + { + states += 25 * BranchBuf[idx]; + + ulong State[8] = { 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0x0001000000000000UL }; + ulong H[8], M[8]; + + // BUG: AMD driver 19.7.X crashs if this is written as loop + // Thx AMD for so bad software + { + ((ulong8 *)M)[0] = vload8(0, states); + + for (uint x = 0; x < 8; ++x) { + H[x] = M[x] ^ State[x]; + } + + PERM_SMALL_P(H); + PERM_SMALL_Q(M); + + for (uint x = 0; x < 8; ++x) { + State[x] ^= H[x] ^ M[x]; + } + } + + { + ((ulong8 *)M)[0] = vload8(1, states); + + for (uint x = 0; x < 8; ++x) { + H[x] = M[x] ^ State[x]; + } + + PERM_SMALL_P(H); + PERM_SMALL_Q(M); + + for (uint x = 0; x < 8; ++x) { + State[x] ^= H[x] ^ M[x]; + } + } + + { + ((ulong8 *)M)[0] = vload8(2, states); + + for (uint x = 0; x < 8; ++x) { + H[x] = M[x] ^ State[x]; + } + + PERM_SMALL_P(H); + PERM_SMALL_Q(M); + + for (uint x = 0; x < 8; ++x) { + State[x] ^= H[x] ^ M[x]; + } + } + + M[0] = states[24]; + M[1] = 0x80UL; + M[2] = 0UL; + M[3] = 0UL; + M[4] = 0UL; + M[5] = 0UL; + M[6] = 0UL; + M[7] = 0x0400000000000000UL; + + for (uint x = 0; x < 8; ++x) { + H[x] = M[x] ^ State[x]; + } + + PERM_SMALL_P(H); + PERM_SMALL_Q(M); + + ulong tmp[8]; + for (uint i = 0; i < 8; ++i) { + tmp[i] = State[i] ^= H[i] ^ M[i]; + } + + PERM_SMALL_P(State); + + for (uint i = 0; i < 8; ++i) { + State[i] ^= tmp[i]; + } + + // Note that comparison is equivalent to subtraction - we can't just compare 8 32-bit values + // and expect an accurate result for target > 32-bit without implementing carries + if (State[7] <= Target) { + ulong outIdx = atomic_inc(output + 0xFF); + if (outIdx < 0xFF) { + output[outIdx] = BranchBuf[idx] + (uint) get_global_offset(0); + } + } + } +} + +)===" diff --git a/src/backend/opencl/cl/cn/cryptonight2.cl b/src/backend/opencl/cl/cn/cryptonight2.cl new file mode 100644 index 00000000..ce3ee019 --- /dev/null +++ b/src/backend/opencl/cl/cn/cryptonight2.cl @@ -0,0 +1,480 @@ +R"===( + +__attribute__((reqd_work_group_size(WORKSIZE, 1, 1))) +__kernel void cn1_v2_rwz(__global uint4 *Scratchpad, __global ulong *states, uint variant, __global ulong *input, uint Threads) +{ +# if (ALGO == CRYPTONIGHT) + ulong a[2], b[4]; + __local uint AES0[256], AES1[256], AES2[256], AES3[256]; + + const ulong gIdx = getIdx(); + + for(int i = get_local_id(0); i < 256; i += WORKSIZE) + { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + AES2[i] = rotate(tmp, 16U); + AES3[i] = rotate(tmp, 24U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; + +# if defined(__NV_CL_C_VERSION) + Scratchpad += gIdx * (0x40000 >> 2); +# else +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) + Scratchpad += gIdx; +# elif (STRIDED_INDEX == 2) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * get_local_id(0); +# endif +# endif + + a[0] = states[0] ^ states[4]; + a[1] = states[1] ^ states[5]; + + b[0] = states[2] ^ states[6]; + b[1] = states[3] ^ states[7]; + b[2] = states[8] ^ states[10]; + b[3] = states[9] ^ states[11]; + } + + ulong2 bx0 = ((ulong2 *)b)[0]; + ulong2 bx1 = ((ulong2 *)b)[1]; + + mem_fence(CLK_LOCAL_MEM_FENCE); + +# ifdef __NV_CL_C_VERSION + __local uint16 scratchpad_line_buf[WORKSIZE]; + __local uint16* scratchpad_line = scratchpad_line_buf + get_local_id(0); +# define SCRATCHPAD_CHUNK(N) (*(__local uint4*)((__local uchar*)(scratchpad_line) + (idx1 ^ (N << 4)))) +# else +# if (STRIDED_INDEX == 0) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + (idx ^ (N << 4)))) +# elif (STRIDED_INDEX == 1) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + mul24(as_uint(idx ^ (N << 4)), Threads))) +# elif (STRIDED_INDEX == 2) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + (((idx ^ (N << 4)) % (MEM_CHUNK << 4)) + ((idx ^ (N << 4)) / (MEM_CHUNK << 4)) * WORKSIZE * (MEM_CHUNK << 4)))) +# endif +# endif + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + uint2 division_result = as_uint2(states[12]); + uint sqrt_result = as_uint2(states[13]).s0; + + #pragma unroll UNROLL_FACTOR + for(int i = 0; i < 0x60000; ++i) + { +# ifdef __NV_CL_C_VERSION + uint idx = a[0] & 0x1FFFC0; + uint idx1 = a[0] & 0x30; + + *scratchpad_line = *(__global uint16*)((__global uchar*)(Scratchpad) + idx); +# else + uint idx = a[0] & MASK; +# endif + + uint4 c = SCRATCHPAD_CHUNK(0); + c = AES_Round(AES0, AES1, AES2, AES3, c, ((uint4 *)a)[0]); + + { + const ulong2 chunk1 = as_ulong2(SCRATCHPAD_CHUNK(3)); + const ulong2 chunk2 = as_ulong2(SCRATCHPAD_CHUNK(2)); + const ulong2 chunk3 = as_ulong2(SCRATCHPAD_CHUNK(1)); + + SCRATCHPAD_CHUNK(1) = as_uint4(chunk3 + bx1); + SCRATCHPAD_CHUNK(2) = as_uint4(chunk1 + bx0); + SCRATCHPAD_CHUNK(3) = as_uint4(chunk2 + ((ulong2 *)a)[0]); + } + + SCRATCHPAD_CHUNK(0) = as_uint4(bx0) ^ c; + +# ifdef __NV_CL_C_VERSION + *(__global uint16*)((__global uchar*)(Scratchpad) + idx) = *scratchpad_line; + + idx = as_ulong2(c).s0 & 0x1FFFC0; + idx1 = as_ulong2(c).s0 & 0x30; + + *scratchpad_line = *(__global uint16*)((__global uchar*)(Scratchpad) + idx); +# else + idx = as_ulong2(c).s0 & MASK; +# endif + + uint4 tmp = SCRATCHPAD_CHUNK(0); + + { + tmp.s0 ^= division_result.s0; + tmp.s1 ^= division_result.s1 ^ sqrt_result; + + division_result = fast_div_v2(as_ulong2(c).s1, (c.s0 + (sqrt_result << 1)) | 0x80000001UL); + sqrt_result = fast_sqrt_v2(as_ulong2(c).s0 + as_ulong(division_result)); + } + + ulong2 t; + t.s0 = mul_hi(as_ulong2(c).s0, as_ulong2(tmp).s0); + t.s1 = as_ulong2(c).s0 * as_ulong2(tmp).s0; + { + const ulong2 chunk1 = as_ulong2(SCRATCHPAD_CHUNK(1)) ^ t; + const ulong2 chunk2 = as_ulong2(SCRATCHPAD_CHUNK(2)); + t ^= chunk2; + const ulong2 chunk3 = as_ulong2(SCRATCHPAD_CHUNK(3)); + + SCRATCHPAD_CHUNK(1) = as_uint4(chunk1 + bx1); + SCRATCHPAD_CHUNK(2) = as_uint4(chunk3 + bx0); + SCRATCHPAD_CHUNK(3) = as_uint4(chunk2 + ((ulong2 *)a)[0]); + } + + a[1] += t.s1; + a[0] += t.s0; + + SCRATCHPAD_CHUNK(0) = ((uint4 *)a)[0]; + +# ifdef __NV_CL_C_VERSION + *(__global uint16*)((__global uchar*)(Scratchpad) + idx) = *scratchpad_line; +# endif + + ((uint4 *)a)[0] ^= tmp; + bx1 = bx0; + bx0 = as_ulong2(c); + } + +# undef SCRATCHPAD_CHUNK + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +# endif +} + +)===" +R"===( + +__attribute__((reqd_work_group_size(WORKSIZE, 1, 1))) +__kernel void cn1_v2_zls(__global uint4 *Scratchpad, __global ulong *states, uint variant, __global ulong *input, uint Threads) +{ +# if (ALGO == CRYPTONIGHT) + ulong a[2], b[4]; + __local uint AES0[256], AES1[256], AES2[256], AES3[256]; + + const ulong gIdx = getIdx(); + + for(int i = get_local_id(0); i < 256; i += WORKSIZE) + { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + AES2[i] = rotate(tmp, 16U); + AES3[i] = rotate(tmp, 24U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; + +# if defined(__NV_CL_C_VERSION) + Scratchpad += gIdx * (0x60000 >> 2); +# else +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) + Scratchpad += gIdx; +# elif (STRIDED_INDEX == 2) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * get_local_id(0); +# endif +# endif + + a[0] = states[0] ^ states[4]; + a[1] = states[1] ^ states[5]; + + b[0] = states[2] ^ states[6]; + b[1] = states[3] ^ states[7]; + b[2] = states[8] ^ states[10]; + b[3] = states[9] ^ states[11]; + } + + ulong2 bx0 = ((ulong2 *)b)[0]; + ulong2 bx1 = ((ulong2 *)b)[1]; + + mem_fence(CLK_LOCAL_MEM_FENCE); + +# ifdef __NV_CL_C_VERSION + __local uint16 scratchpad_line_buf[WORKSIZE]; + __local uint16* scratchpad_line = scratchpad_line_buf + get_local_id(0); +# define SCRATCHPAD_CHUNK(N) (*(__local uint4*)((__local uchar*)(scratchpad_line) + (idx1 ^ (N << 4)))) +# else +# if (STRIDED_INDEX == 0) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + (idx ^ (N << 4)))) +# elif (STRIDED_INDEX == 1) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + mul24(as_uint(idx ^ (N << 4)), Threads))) +# elif (STRIDED_INDEX == 2) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + (((idx ^ (N << 4)) % (MEM_CHUNK << 4)) + ((idx ^ (N << 4)) / (MEM_CHUNK << 4)) * WORKSIZE * (MEM_CHUNK << 4)))) +# endif +# endif + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + uint2 division_result = as_uint2(states[12]); + uint sqrt_result = as_uint2(states[13]).s0; + + #pragma unroll UNROLL_FACTOR + for(int i = 0; i < 0x60000; ++i) + { +# ifdef __NV_CL_C_VERSION + uint idx = a[0] & 0x1FFFC0; + uint idx1 = a[0] & 0x30; + + *scratchpad_line = *(__global uint16*)((__global uchar*)(Scratchpad) + idx); +# else + uint idx = a[0] & MASK; +# endif + + uint4 c = SCRATCHPAD_CHUNK(0); + c = AES_Round(AES0, AES1, AES2, AES3, c, ((uint4 *)a)[0]); + + { + const ulong2 chunk1 = as_ulong2(SCRATCHPAD_CHUNK(1)); + const ulong2 chunk2 = as_ulong2(SCRATCHPAD_CHUNK(2)); + const ulong2 chunk3 = as_ulong2(SCRATCHPAD_CHUNK(3)); + + SCRATCHPAD_CHUNK(1) = as_uint4(chunk3 + bx1); + SCRATCHPAD_CHUNK(2) = as_uint4(chunk1 + bx0); + SCRATCHPAD_CHUNK(3) = as_uint4(chunk2 + ((ulong2 *)a)[0]); + } + + SCRATCHPAD_CHUNK(0) = as_uint4(bx0) ^ c; + +# ifdef __NV_CL_C_VERSION + *(__global uint16*)((__global uchar*)(Scratchpad) + idx) = *scratchpad_line; + + idx = as_ulong2(c).s0 & 0x1FFFC0; + idx1 = as_ulong2(c).s0 & 0x30; + + *scratchpad_line = *(__global uint16*)((__global uchar*)(Scratchpad) + idx); +# else + idx = as_ulong2(c).s0 & MASK; +# endif + + uint4 tmp = SCRATCHPAD_CHUNK(0); + + { + tmp.s0 ^= division_result.s0; + tmp.s1 ^= division_result.s1 ^ sqrt_result; + + division_result = fast_div_v2(as_ulong2(c).s1, (c.s0 + (sqrt_result << 1)) | 0x80000001UL); + sqrt_result = fast_sqrt_v2(as_ulong2(c).s0 + as_ulong(division_result)); + } + + ulong2 t; + t.s0 = mul_hi(as_ulong2(c).s0, as_ulong2(tmp).s0); + t.s1 = as_ulong2(c).s0 * as_ulong2(tmp).s0; + { + const ulong2 chunk1 = as_ulong2(SCRATCHPAD_CHUNK(1)) ^ t; + const ulong2 chunk2 = as_ulong2(SCRATCHPAD_CHUNK(2)); + t ^= chunk2; + const ulong2 chunk3 = as_ulong2(SCRATCHPAD_CHUNK(3)); + + SCRATCHPAD_CHUNK(1) = as_uint4(chunk3 + bx1); + SCRATCHPAD_CHUNK(2) = as_uint4(chunk1 + bx0); + SCRATCHPAD_CHUNK(3) = as_uint4(chunk2 + ((ulong2 *)a)[0]); + } + + a[1] += t.s1; + a[0] += t.s0; + + SCRATCHPAD_CHUNK(0) = ((uint4 *)a)[0]; + +# ifdef __NV_CL_C_VERSION + *(__global uint16*)((__global uchar*)(Scratchpad) + idx) = *scratchpad_line; +# endif + + ((uint4 *)a)[0] ^= tmp; + bx1 = bx0; + bx0 = as_ulong2(c); + } + +# undef SCRATCHPAD_CHUNK + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +# endif +} + +)===" +R"===( + +__attribute__((reqd_work_group_size(WORKSIZE, 1, 1))) +__kernel void cn1_v2_double(__global uint4 *Scratchpad, __global ulong *states, uint variant, __global ulong *input, uint Threads) +{ +# if (ALGO == CRYPTONIGHT) + ulong a[2], b[4]; + __local uint AES0[256], AES1[256], AES2[256], AES3[256]; + + const ulong gIdx = getIdx(); + + for(int i = get_local_id(0); i < 256; i += WORKSIZE) + { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + AES2[i] = rotate(tmp, 16U); + AES3[i] = rotate(tmp, 24U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; + +# if defined(__NV_CL_C_VERSION) + Scratchpad += gIdx * (0x100000 >> 2); +# else +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) + Scratchpad += gIdx; +# elif (STRIDED_INDEX == 2) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * get_local_id(0); +# endif +# endif + + a[0] = states[0] ^ states[4]; + a[1] = states[1] ^ states[5]; + + b[0] = states[2] ^ states[6]; + b[1] = states[3] ^ states[7]; + b[2] = states[8] ^ states[10]; + b[3] = states[9] ^ states[11]; + } + + ulong2 bx0 = ((ulong2 *)b)[0]; + ulong2 bx1 = ((ulong2 *)b)[1]; + + mem_fence(CLK_LOCAL_MEM_FENCE); + +# ifdef __NV_CL_C_VERSION + __local uint16 scratchpad_line_buf[WORKSIZE]; + __local uint16* scratchpad_line = scratchpad_line_buf + get_local_id(0); +# define SCRATCHPAD_CHUNK(N) (*(__local uint4*)((__local uchar*)(scratchpad_line) + (idx1 ^ (N << 4)))) +# else +# if (STRIDED_INDEX == 0) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + (idx ^ (N << 4)))) +# elif (STRIDED_INDEX == 1) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + mul24(as_uint(idx ^ (N << 4)), Threads))) +# elif (STRIDED_INDEX == 2) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + (((idx ^ (N << 4)) % (MEM_CHUNK << 4)) + ((idx ^ (N << 4)) / (MEM_CHUNK << 4)) * WORKSIZE * (MEM_CHUNK << 4)))) +# endif +# endif + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + uint2 division_result = as_uint2(states[12]); + uint sqrt_result = as_uint2(states[13]).s0; + + #pragma unroll UNROLL_FACTOR + for(int i = 0; i < 0x100000; ++i) + { +# ifdef __NV_CL_C_VERSION + uint idx = a[0] & 0x1FFFC0; + uint idx1 = a[0] & 0x30; + + *scratchpad_line = *(__global uint16*)((__global uchar*)(Scratchpad) + idx); +# else + uint idx = a[0] & MASK; +# endif + + uint4 c = SCRATCHPAD_CHUNK(0); + c = AES_Round(AES0, AES1, AES2, AES3, c, ((uint4 *)a)[0]); + + { + const ulong2 chunk1 = as_ulong2(SCRATCHPAD_CHUNK(1)); + const ulong2 chunk2 = as_ulong2(SCRATCHPAD_CHUNK(2)); + const ulong2 chunk3 = as_ulong2(SCRATCHPAD_CHUNK(3)); + + SCRATCHPAD_CHUNK(1) = as_uint4(chunk3 + bx1); + SCRATCHPAD_CHUNK(2) = as_uint4(chunk1 + bx0); + SCRATCHPAD_CHUNK(3) = as_uint4(chunk2 + ((ulong2 *)a)[0]); + } + + SCRATCHPAD_CHUNK(0) = as_uint4(bx0) ^ c; + +# ifdef __NV_CL_C_VERSION + *(__global uint16*)((__global uchar*)(Scratchpad) + idx) = *scratchpad_line; + + idx = as_ulong2(c).s0 & 0x1FFFC0; + idx1 = as_ulong2(c).s0 & 0x30; + + *scratchpad_line = *(__global uint16*)((__global uchar*)(Scratchpad) + idx); +# else + idx = as_ulong2(c).s0 & MASK; +# endif + + uint4 tmp = SCRATCHPAD_CHUNK(0); + + { + tmp.s0 ^= division_result.s0; + tmp.s1 ^= division_result.s1 ^ sqrt_result; + + division_result = fast_div_v2(as_ulong2(c).s1, (c.s0 + (sqrt_result << 1)) | 0x80000001UL); + sqrt_result = fast_sqrt_v2(as_ulong2(c).s0 + as_ulong(division_result)); + } + + ulong2 t; + t.s0 = mul_hi(as_ulong2(c).s0, as_ulong2(tmp).s0); + t.s1 = as_ulong2(c).s0 * as_ulong2(tmp).s0; + { + const ulong2 chunk1 = as_ulong2(SCRATCHPAD_CHUNK(1)) ^ t; + const ulong2 chunk2 = as_ulong2(SCRATCHPAD_CHUNK(2)); + t ^= chunk2; + const ulong2 chunk3 = as_ulong2(SCRATCHPAD_CHUNK(3)); + + SCRATCHPAD_CHUNK(1) = as_uint4(chunk3 + bx1); + SCRATCHPAD_CHUNK(2) = as_uint4(chunk1 + bx0); + SCRATCHPAD_CHUNK(3) = as_uint4(chunk2 + ((ulong2 *)a)[0]); + } + + a[1] += t.s1; + a[0] += t.s0; + + SCRATCHPAD_CHUNK(0) = ((uint4 *)a)[0]; + +# ifdef __NV_CL_C_VERSION + *(__global uint16*)((__global uchar*)(Scratchpad) + idx) = *scratchpad_line; +# endif + + ((uint4 *)a)[0] ^= tmp; + bx1 = bx0; + bx0 = as_ulong2(c); + } + +# undef SCRATCHPAD_CHUNK + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +# endif +} + +)===" diff --git a/src/backend/opencl/cl/cn/cryptonight_gpu.cl b/src/backend/opencl/cl/cn/cryptonight_gpu.cl new file mode 100644 index 00000000..ee39f6ca --- /dev/null +++ b/src/backend/opencl/cl/cn/cryptonight_gpu.cl @@ -0,0 +1,526 @@ +R"===( + + +inline float4 _mm_add_ps(float4 a, float4 b) +{ + return a + b; +} + +inline float4 _mm_sub_ps(float4 a, float4 b) +{ + return a - b; +} + +inline float4 _mm_mul_ps(float4 a, float4 b) +{ + return a * b; +} + +inline float4 _mm_div_ps(float4 a, float4 b) +{ + return a / b; +} + +inline float4 _mm_and_ps(float4 a, int b) +{ + return as_float4(as_int4(a) & (int4)(b)); +} + +inline float4 _mm_or_ps(float4 a, int b) +{ + return as_float4(as_int4(a) | (int4)(b)); +} + +inline float4 _mm_fmod_ps(float4 v, float dc) +{ + float4 d = (float4)(dc); + float4 c = _mm_div_ps(v, d); + c = trunc(c); + c = _mm_mul_ps(c, d); + return _mm_sub_ps(v, c); +} + +inline int4 _mm_xor_si128(int4 a, int4 b) +{ + return a ^ b; +} + +inline float4 _mm_xor_ps(float4 a, int b) +{ + return as_float4(as_int4(a) ^ (int4)(b)); +} + +inline int4 _mm_alignr_epi8(int4 a, const uint rot) +{ + const uint right = 8 * rot; + const uint left = (32 - 8 * rot); + return (int4)( + ((uint)a.x >> right) | ( a.y << left ), + ((uint)a.y >> right) | ( a.z << left ), + ((uint)a.z >> right) | ( a.w << left ), + ((uint)a.w >> right) | ( a.x << left ) + ); +} + + +inline global int4* scratchpad_ptr(uint idx, uint n, __global int *lpad) { return (__global int4*)((__global char*)lpad + (idx & 0x1FFFC0) + n * 16); } + +inline float4 fma_break(float4 x) +{ + // Break the dependency chain by setitng the exp to ?????01 + x = _mm_and_ps(x, 0xFEFFFFFF); + return _mm_or_ps(x, 0x00800000); +} + +inline void sub_round(float4 n0, float4 n1, float4 n2, float4 n3, float4 rnd_c, float4* n, float4* d, float4* c) +{ + n1 = _mm_add_ps(n1, *c); + float4 nn = _mm_mul_ps(n0, *c); + nn = _mm_mul_ps(n1, _mm_mul_ps(nn,nn)); + nn = fma_break(nn); + *n = _mm_add_ps(*n, nn); + + n3 = _mm_sub_ps(n3, *c); + float4 dd = _mm_mul_ps(n2, *c); + dd = _mm_mul_ps(n3, _mm_mul_ps(dd,dd)); + dd = fma_break(dd); + *d = _mm_add_ps(*d, dd); + + //Constant feedback + *c = _mm_add_ps(*c, rnd_c); + *c = _mm_add_ps(*c, (float4)(0.734375f)); + float4 r = _mm_add_ps(nn, dd); + r = _mm_and_ps(r, 0x807FFFFF); + r = _mm_or_ps(r, 0x40000000); + *c = _mm_add_ps(*c, r); + +} + +// 9*8 + 2 = 74 +inline void round_compute(float4 n0, float4 n1, float4 n2, float4 n3, float4 rnd_c, float4* c, float4* r) +{ + float4 n = (float4)(0.0f); + float4 d = (float4)(0.0f); + + sub_round(n0, n1, n2, n3, rnd_c, &n, &d, c); + sub_round(n1, n2, n3, n0, rnd_c, &n, &d, c); + sub_round(n2, n3, n0, n1, rnd_c, &n, &d, c); + sub_round(n3, n0, n1, n2, rnd_c, &n, &d, c); + sub_round(n3, n2, n1, n0, rnd_c, &n, &d, c); + sub_round(n2, n1, n0, n3, rnd_c, &n, &d, c); + sub_round(n1, n0, n3, n2, rnd_c, &n, &d, c); + sub_round(n0, n3, n2, n1, rnd_c, &n, &d, c); + + // Make sure abs(d) > 2.0 - this prevents division by zero and accidental overflows by division by < 1.0 + d = _mm_and_ps(d, 0xFF7FFFFF); + d = _mm_or_ps(d, 0x40000000); + *r =_mm_add_ps(*r, _mm_div_ps(n,d)); +} + +inline int4 single_comupte(float4 n0, float4 n1, float4 n2, float4 n3, float cnt, float4 rnd_c, __local float4* sum) +{ + float4 c= (float4)(cnt); + // 35 maths calls follow (140 FLOPS) + float4 r = (float4)(0.0f); + + for(int i = 0; i < 4; ++i) + round_compute(n0, n1, n2, n3, rnd_c, &c, &r); + + // do a quick fmod by setting exp to 2 + r = _mm_and_ps(r, 0x807FFFFF); + r = _mm_or_ps(r, 0x40000000); + *sum = r; // 34 + float4 x = (float4)(536870880.0f); + r = _mm_mul_ps(r, x); // 35 + return convert_int4_rte(r); +} + +inline void single_comupte_wrap(const uint rot, int4 v0, int4 v1, int4 v2, int4 v3, float cnt, float4 rnd_c, __local float4* sum, __local int4* out) +{ + float4 n0 = convert_float4_rte(v0); + float4 n1 = convert_float4_rte(v1); + float4 n2 = convert_float4_rte(v2); + float4 n3 = convert_float4_rte(v3); + + int4 r = single_comupte(n0, n1, n2, n3, cnt, rnd_c, sum); + *out = rot == 0 ? r : _mm_alignr_epi8(r, rot); +} + +)===" +R"===( + +static const __constant uint look[16][4] = { + {0, 1, 2, 3}, + {0, 2, 3, 1}, + {0, 3, 1, 2}, + {0, 3, 2, 1}, + + {1, 0, 2, 3}, + {1, 2, 3, 0}, + {1, 3, 0, 2}, + {1, 3, 2, 0}, + + {2, 1, 0, 3}, + {2, 0, 3, 1}, + {2, 3, 1, 0}, + {2, 3, 0, 1}, + + {3, 1, 2, 0}, + {3, 2, 0, 1}, + {3, 0, 1, 2}, + {3, 0, 2, 1} +}; + +static const __constant float ccnt[16] = { + 1.34375f, + 1.28125f, + 1.359375f, + 1.3671875f, + + 1.4296875f, + 1.3984375f, + 1.3828125f, + 1.3046875f, + + 1.4140625f, + 1.2734375f, + 1.2578125f, + 1.2890625f, + + 1.3203125f, + 1.3515625f, + 1.3359375f, + 1.4609375f +}; + +struct SharedMemChunk +{ + int4 out[16]; + float4 va[16]; +}; + +__attribute__((reqd_work_group_size(WORKSIZE_GPU * 16, 1, 1))) +__kernel void cn1_cn_gpu(__global int *lpad_in, __global int *spad, uint numThreads) +{ + const uint gIdx = getIdx(); + +# if (COMP_MODE==1) + if (gIdx / 16 >= numThreads) { + return; + } +# endif + + uint chunk = get_local_id(0) / 16; + + __global int* lpad = (__global int*)((__global char*)lpad_in + MEMORY * (gIdx/16)); + + __local struct SharedMemChunk smem_in[WORKSIZE_GPU]; + __local struct SharedMemChunk* smem = smem_in + chunk; + + uint tid = get_local_id(0) % 16; + + uint idxHash = gIdx/16; + uint s = ((__global uint*)spad)[idxHash * 50] >> 8; + float4 vs = (float4)(0); + + // tid divided + const uint tidd = tid / 4; + // tid modulo + const uint tidm = tid % 4; + const uint block = tidd * 16 + tidm; + + #pragma unroll 1 + for(size_t i = 0; i < 0xC000; i++) + { + mem_fence(CLK_LOCAL_MEM_FENCE); + int tmp = ((__global int*)scratchpad_ptr(s, tidd, lpad))[tidm]; + ((__local int*)(smem->out))[tid] = tmp; + mem_fence(CLK_LOCAL_MEM_FENCE); + + { + single_comupte_wrap( + tidm, + *(smem->out + look[tid][0]), + *(smem->out + look[tid][1]), + *(smem->out + look[tid][2]), + *(smem->out + look[tid][3]), + ccnt[tid], vs, smem->va + tid, + smem->out + tid + ); + } + mem_fence(CLK_LOCAL_MEM_FENCE); + + int outXor = ((__local int*)smem->out)[block]; + for(uint dd = block + 4; dd < (tidd + 1) * 16; dd += 4) + outXor ^= ((__local int*)smem->out)[dd]; + + ((__global int*)scratchpad_ptr(s, tidd, lpad))[tidm] = outXor ^ tmp; + ((__local int*)smem->out)[tid] = outXor; + + float va_tmp1 = ((__local float*)smem->va)[block] + ((__local float*)smem->va)[block + 4]; + float va_tmp2 = ((__local float*)smem->va)[block+ 8] + ((__local float*)smem->va)[block + 12]; + ((__local float*)smem->va)[tid] = va_tmp1 + va_tmp2; + + mem_fence(CLK_LOCAL_MEM_FENCE); + + int out2 = ((__local int*)smem->out)[tid] ^ ((__local int*)smem->out)[tid + 4 ] ^ ((__local int*)smem->out)[tid + 8] ^ ((__local int*)smem->out)[tid + 12]; + va_tmp1 = ((__local float*)smem->va)[block] + ((__local float*)smem->va)[block + 4]; + va_tmp2 = ((__local float*)smem->va)[block + 8] + ((__local float*)smem->va)[block + 12]; + va_tmp1 = va_tmp1 + va_tmp2; + va_tmp1 = fabs(va_tmp1); + + float xx = va_tmp1 * 16777216.0f; + int xx_int = (int)xx; + ((__local int*)smem->out)[tid] = out2 ^ xx_int; + ((__local float*)smem->va)[tid] = va_tmp1 / 64.0f; + + mem_fence(CLK_LOCAL_MEM_FENCE); + + vs = smem->va[0]; + s = smem->out[0].x ^ smem->out[0].y ^ smem->out[0].z ^ smem->out[0].w; + } +} + +)===" +R"===( + +static const __constant uint skip[3] = { + 20,22,22 +}; + +inline void generate_512(uint idx, __local ulong* in, __global ulong* out) +{ + ulong hash[25]; + + hash[0] = in[0] ^ idx; + for(int i = 1; i < 25; ++i) + hash[i] = in[i]; + + for(int a = 0; a < 3;++a) + { + keccakf1600_1(hash); + for(int i = 0; i < skip[a]; ++i) + out[i] = hash[i]; + out+=skip[a]; + } +} + +__attribute__((reqd_work_group_size(8, 8, 1))) +__kernel void cn0_cn_gpu(__global ulong *input, __global int *Scratchpad, __global ulong *states, uint Threads) +{ + const uint gIdx = getIdx(); + __local ulong State_buf[8 * 25]; + __local ulong* State = State_buf + get_local_id(0) * 25; + +# if (COMP_MODE==1) + // do not use early return here + if(gIdx < Threads) +# endif + { + states += 25 * gIdx; + + Scratchpad = (__global int*)((__global char*)Scratchpad + MEMORY * gIdx); + + if (get_local_id(1) == 0) + { + +// NVIDIA +#ifdef __NV_CL_C_VERSION + for(uint i = 0; i < 8; ++i) + State[i] = input[i]; +#else + ((__local ulong8 *)State)[0] = vload8(0, input); +#endif + State[8] = input[8]; + State[9] = input[9]; + State[10] = input[10]; + + ((__local uint *)State)[9] &= 0x00FFFFFFU; + ((__local uint *)State)[9] |= (((uint)get_global_id(0)) & 0xFF) << 24; + ((__local uint *)State)[10] &= 0xFF000000U; + /* explicit cast to `uint` is required because some OpenCL implementations (e.g. NVIDIA) + * handle get_global_id and get_global_offset as signed long long int and add + * 0xFFFFFFFF... to `get_global_id` if we set on host side a 32bit offset where the first bit is `1` + * (even if it is correct casted to unsigned on the host) + */ + ((__local uint *)State)[10] |= (((uint)get_global_id(0) >> 8)); + + for (int i = 11; i < 25; ++i) { + State[i] = 0x00UL; + } + + // Last bit of padding + State[16] = 0x8000000000000000UL; + + keccakf1600_2(State); + + #pragma unroll + for (int i = 0; i < 25; ++i) { + states[i] = State[i]; + } + } + } +} + +__attribute__((reqd_work_group_size(64, 1, 1))) +__kernel void cn00_cn_gpu(__global int *Scratchpad, __global ulong *states) +{ + const uint gIdx = getIdx() / 64; + __local ulong State[25]; + + states += 25 * gIdx; + + Scratchpad = (__global int*)((__global char*)Scratchpad + MEMORY * gIdx); + + for(int i = get_local_id(0); i < 25; i+=get_local_size(0)) + State[i] = states[i]; + + barrier(CLK_LOCAL_MEM_FENCE); + + + for(uint i = get_local_id(0); i < MEMORY / 512; i += get_local_size(0)) + { + generate_512(i, State, (__global ulong*)((__global uchar*)Scratchpad + i*512)); + } +} + +__attribute__((reqd_work_group_size(8, 8, 1))) +__kernel void cn2_cn_gpu(__global uint4 *Scratchpad, __global ulong *states, __global uint *output, ulong Target, uint Threads) +{ + __local uint AES0[256], AES1[256], AES2[256], AES3[256]; + uint ExpandedKey2[40]; + uint4 text; + + const uint gIdx = getIdx(); + + for (int i = get_local_id(1) * 8 + get_local_id(0); i < 256; i += 8 * 8) { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + AES2[i] = rotate(tmp, 16U); + AES3[i] = rotate(tmp, 24U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + + __local uint4 xin1[8][8]; + __local uint4 xin2[8][8]; + +# if (COMP_MODE==1) + // do not use early return here + if(gIdx < Threads) +# endif + { + states += 25 * gIdx; + Scratchpad += gIdx * (MEMORY >> 4); + + #if defined(__Tahiti__) || defined(__Pitcairn__) + + for(int i = 0; i < 4; ++i) ((ulong *)ExpandedKey2)[i] = states[i + 4]; + text = vload4(get_local_id(1) + 4, (__global uint *)states); + + #else + + text = vload4(get_local_id(1) + 4, (__global uint *)states); + ((uint8 *)ExpandedKey2)[0] = vload8(1, (__global uint *)states); + + #endif + + AESExpandKey256(ExpandedKey2); + } + + barrier(CLK_LOCAL_MEM_FENCE); + + __local uint4* xin1_store = &xin1[get_local_id(1)][get_local_id(0)]; + __local uint4* xin1_load = &xin1[(get_local_id(1) + 1) % 8][get_local_id(0)]; + __local uint4* xin2_store = &xin2[get_local_id(1)][get_local_id(0)]; + __local uint4* xin2_load = &xin2[(get_local_id(1) + 1) % 8][get_local_id(0)]; + *xin2_store = (uint4)(0, 0, 0, 0); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + + #pragma unroll 2 + for(int i = 0, i1 = get_local_id(1); i < (MEMORY >> 7); ++i, i1 = (i1 + 16) % (MEMORY >> 4)) + { + text ^= Scratchpad[(uint)i1]; + barrier(CLK_LOCAL_MEM_FENCE); + text ^= *xin2_load; + + #pragma unroll 10 + for(int j = 0; j < 10; ++j) + text = AES_Round(AES0, AES1, AES2, AES3, text, ((uint4 *)ExpandedKey2)[j]); + + *xin1_store = text; + + text ^= Scratchpad[(uint)i1 + 8u]; + barrier(CLK_LOCAL_MEM_FENCE); + text ^= *xin1_load; + + #pragma unroll 10 + for(int j = 0; j < 10; ++j) + text = AES_Round(AES0, AES1, AES2, AES3, text, ((uint4 *)ExpandedKey2)[j]); + + *xin2_store = text; + } + + barrier(CLK_LOCAL_MEM_FENCE); + text ^= *xin2_load; + } + + /* Also left over threads performe this loop. + * The left over thread results will be ignored + */ + #pragma unroll 16 + for(size_t i = 0; i < 16; i++) + { + #pragma unroll 10 + for (int j = 0; j < 10; ++j) { + text = AES_Round(AES0, AES1, AES2, AES3, text, ((uint4 *)ExpandedKey2)[j]); + } + + barrier(CLK_LOCAL_MEM_FENCE); + *xin1_store = text; + barrier(CLK_LOCAL_MEM_FENCE); + text ^= *xin1_load; + } + + __local ulong State_buf[8 * 25]; +# if (COMP_MODE==1) + // do not use early return here + if(gIdx < Threads) +# endif + { + vstore2(as_ulong2(text), get_local_id(1) + 4, states); + } + + barrier(CLK_GLOBAL_MEM_FENCE); + +# if (COMP_MODE==1) + // do not use early return here + if(gIdx < Threads) +# endif + { + if(!get_local_id(1)) + { + __local ulong* State = State_buf + get_local_id(0) * 25; + + for(int i = 0; i < 25; ++i) State[i] = states[i]; + + keccakf1600_2(State); + + if(State[3] <= Target) + { + ulong outIdx = atomic_inc(output + 0xFF); + if(outIdx < 0xFF) + output[outIdx] = get_global_id(0); + } + } + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +} + +)===" diff --git a/src/backend/opencl/cl/cn/cryptonight_r.cl b/src/backend/opencl/cl/cn/cryptonight_r.cl new file mode 100644 index 00000000..857318b6 --- /dev/null +++ b/src/backend/opencl/cl/cn/cryptonight_r.cl @@ -0,0 +1,175 @@ +R"===( +__attribute__((reqd_work_group_size(WORKSIZE, 1, 1))) +__kernel void cn1_cryptonight_r_N(__global uint4 *Scratchpad, __global ulong *states, uint variant, __global ulong *input, uint Threads) +{ + ulong a[2], b[4]; + __local uint AES0[256], AES1[256], AES2[256], AES3[256]; + + const ulong gIdx = get_global_id(0) - get_global_offset(0); + + for(int i = get_local_id(0); i < 256; i += WORKSIZE) + { + const uint tmp = AES0_C[i]; + AES0[i] = tmp; + AES1[i] = rotate(tmp, 8U); + AES2[i] = rotate(tmp, 16U); + AES3[i] = rotate(tmp, 24U); + } + + barrier(CLK_LOCAL_MEM_FENCE); + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + states += 25 * gIdx; + +# if defined(__NV_CL_C_VERSION) + Scratchpad += gIdx * (ITERATIONS >> 2); +# else +# if (STRIDED_INDEX == 0) + Scratchpad += gIdx * (MEMORY >> 4); +# elif (STRIDED_INDEX == 1) + Scratchpad += gIdx; +# elif (STRIDED_INDEX == 2) + Scratchpad += get_group_id(0) * (MEMORY >> 4) * WORKSIZE + MEM_CHUNK * get_local_id(0); +# endif +# endif + + a[0] = states[0] ^ states[4]; + a[1] = states[1] ^ states[5]; + + b[0] = states[2] ^ states[6]; + b[1] = states[3] ^ states[7]; + b[2] = states[8] ^ states[10]; + b[3] = states[9] ^ states[11]; + } + + ulong2 bx0 = ((ulong2 *)b)[0]; + ulong2 bx1 = ((ulong2 *)b)[1]; + + mem_fence(CLK_LOCAL_MEM_FENCE); + +# ifdef __NV_CL_C_VERSION + __local uint16 scratchpad_line_buf[WORKSIZE]; + __local uint16* scratchpad_line = scratchpad_line_buf + get_local_id(0); +# endif + +# if (COMP_MODE == 1) + // do not use early return here + if (gIdx < Threads) +# endif + { + uint r0 = as_uint2(states[12]).s0; + uint r1 = as_uint2(states[12]).s1; + uint r2 = as_uint2(states[13]).s0; + uint r3 = as_uint2(states[13]).s1; + + #pragma unroll UNROLL_FACTOR + for(int i = 0; i < ITERATIONS; ++i) + { +# ifdef __NV_CL_C_VERSION + uint idx = a[0] & 0x1FFFC0; + uint idx1 = a[0] & 0x30; + + *scratchpad_line = *(__global uint16*)((__global uchar*)(Scratchpad) + idx); +# else + uint idx = a[0] & MASK; +# endif + + uint4 c = SCRATCHPAD_CHUNK(0); + c = AES_Round(AES0, AES1, AES2, AES3, c, ((uint4 *)a)[0]); + + { + const ulong2 chunk1 = as_ulong2(SCRATCHPAD_CHUNK(1)); + const ulong2 chunk2 = as_ulong2(SCRATCHPAD_CHUNK(2)); + const ulong2 chunk3 = as_ulong2(SCRATCHPAD_CHUNK(3)); + +#if (VARIANT == VARIANT_4) + c ^= as_uint4(chunk1) ^ as_uint4(chunk2) ^ as_uint4(chunk3); +#endif + + SCRATCHPAD_CHUNK(1) = as_uint4(chunk3 + bx1); + SCRATCHPAD_CHUNK(2) = as_uint4(chunk1 + bx0); + SCRATCHPAD_CHUNK(3) = as_uint4(chunk2 + ((ulong2 *)a)[0]); + } + + SCRATCHPAD_CHUNK(0) = as_uint4(bx0) ^ c; + +# ifdef __NV_CL_C_VERSION + *(__global uint16*)((__global uchar*)(Scratchpad) + idx) = *scratchpad_line; + + idx = as_ulong2(c).s0 & 0x1FFFC0; + idx1 = as_ulong2(c).s0 & 0x30; + + *scratchpad_line = *(__global uint16*)((__global uchar*)(Scratchpad) + idx); +# else + idx = as_ulong2(c).s0 & MASK; +# endif + + uint4 tmp = SCRATCHPAD_CHUNK(0); + + tmp.s0 ^= r0 + r1; + tmp.s1 ^= r2 + r3; + const uint r4 = as_uint2(a[0]).s0; + const uint r5 = as_uint2(a[1]).s0; + const uint r6 = as_uint4(bx0).s0; + const uint r7 = as_uint4(bx1).s0; +#if (VARIANT == VARIANT_4) + const uint r8 = as_uint4(bx1).s2; +#endif + + XMRIG_INCLUDE_RANDOM_MATH + +#if (VARIANT == VARIANT_4) + const uint2 al = (uint2)(as_uint2(a[0]).s0 ^ r2, as_uint2(a[0]).s1 ^ r3); + const uint2 ah = (uint2)(as_uint2(a[1]).s0 ^ r0, as_uint2(a[1]).s1 ^ r1); +#endif + + ulong2 t; + t.s0 = mul_hi(as_ulong2(c).s0, as_ulong2(tmp).s0); + t.s1 = as_ulong2(c).s0 * as_ulong2(tmp).s0; + { + const ulong2 chunk1 = as_ulong2(SCRATCHPAD_CHUNK(1)) +#if (VARIANT == VARIANT_WOW) + ^ t +#endif + ; + const ulong2 chunk2 = as_ulong2(SCRATCHPAD_CHUNK(2)); +#if (VARIANT == VARIANT_WOW) + t ^= chunk2; +#endif + const ulong2 chunk3 = as_ulong2(SCRATCHPAD_CHUNK(3)); + +#if (VARIANT == VARIANT_4) + c ^= as_uint4(chunk1) ^ as_uint4(chunk2) ^ as_uint4(chunk3); +#endif + + SCRATCHPAD_CHUNK(1) = as_uint4(chunk3 + bx1); + SCRATCHPAD_CHUNK(2) = as_uint4(chunk1 + bx0); + SCRATCHPAD_CHUNK(3) = as_uint4(chunk2 + ((ulong2 *)a)[0]); + } + +#if (VARIANT == VARIANT_4) + a[1] = as_ulong(ah) + t.s1; + a[0] = as_ulong(al) + t.s0; +#else + a[1] += t.s1; + a[0] += t.s0; +#endif + + SCRATCHPAD_CHUNK(0) = ((uint4 *)a)[0]; + +# ifdef __NV_CL_C_VERSION + *(__global uint16*)((__global uchar*)(Scratchpad) + idx) = *scratchpad_line; +# endif + + ((uint4 *)a)[0] ^= tmp; + bx1 = bx0; + bx0 = as_ulong2(c); + } + } + mem_fence(CLK_GLOBAL_MEM_FENCE); +} +)===" diff --git a/src/backend/opencl/cl/cn/cryptonight_r_defines.cl b/src/backend/opencl/cl/cn/cryptonight_r_defines.cl new file mode 100644 index 00000000..03d1bdf5 --- /dev/null +++ b/src/backend/opencl/cl/cn/cryptonight_r_defines.cl @@ -0,0 +1,20 @@ +R"===( +#ifdef __NV_CL_C_VERSION +# define SCRATCHPAD_CHUNK(N) (*(__local uint4*)((__local uchar*)(scratchpad_line) + (idx1 ^ (N << 4)))) +#else +# if (STRIDED_INDEX == 0) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + (idx ^ (N << 4)))) +# elif (STRIDED_INDEX == 1) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + mul24(as_uint(idx ^ (N << 4)), Threads))) +# elif (STRIDED_INDEX == 2) +# define SCRATCHPAD_CHUNK(N) (*(__global uint4*)((__global uchar*)(Scratchpad) + (((idx ^ (N << 4)) % (MEM_CHUNK << 4)) + ((idx ^ (N << 4)) / (MEM_CHUNK << 4)) * WORKSIZE * (MEM_CHUNK << 4)))) +# endif +#endif + +#define ROT_BITS 32 + +#define VARIANT_WOW 12 +#define VARIANT_4 13 + +#define MEM_CHUNK (1 << MEM_CHUNK_EXPONENT) +)===" diff --git a/src/backend/opencl/cl/cn/fast_div_heavy.cl b/src/backend/opencl/cl/cn/fast_div_heavy.cl new file mode 100644 index 00000000..c8fa90c4 --- /dev/null +++ b/src/backend/opencl/cl/cn/fast_div_heavy.cl @@ -0,0 +1,29 @@ +R"===( +#ifndef FAST_DIV_HEAVY_CL +#define FAST_DIV_HEAVY_CL + +inline long fast_div_heavy(long _a, int _b) +{ + long a = abs(_a); + int b = abs(_b); + + float rcp = native_recip(convert_float_rte(b)); + float rcp2 = as_float(as_uint(rcp) + (32U << 23)); + + ulong q1 = convert_ulong(convert_float_rte(as_int2(a).s1) * rcp2); + a -= q1 * as_uint(b); + + float q2f = convert_float_rte(as_int2(a >> 12).s0) * rcp; + q2f = as_float(as_uint(q2f) + (12U << 23)); + long q2 = convert_long_rte(q2f); + int a2 = as_int2(a).s0 - as_int2(q2).s0 * b; + + int q3 = convert_int_rte(convert_float_rte(a2) * rcp); + q3 += (a2 - q3 * b) >> 31; + + const long q = q1 + q2 + q3; + return ((as_int2(_a).s1 ^ _b) < 0) ? -q : q; +} + +#endif +)===" diff --git a/src/backend/opencl/cl/cn/fast_int_math_v2.cl b/src/backend/opencl/cl/cn/fast_int_math_v2.cl new file mode 100644 index 00000000..c840104c --- /dev/null +++ b/src/backend/opencl/cl/cn/fast_int_math_v2.cl @@ -0,0 +1,57 @@ +R"===( +/* + * @author SChernykh + */ + +inline uint get_reciprocal(uint a) +{ + const float a_hi = as_float((a >> 8) + ((126U + 31U) << 23)); + const float a_lo = convert_float_rte(a & 0xFF); + + const float r = native_recip(a_hi); + const float r_scaled = as_float(as_uint(r) + (64U << 23)); + + const float h = fma(a_lo, r, fma(a_hi, r, -1.0f)); + return (as_uint(r) << 9) - convert_int_rte(h * r_scaled); +} + +inline uint2 fast_div_v2(ulong a, uint b) +{ + const uint r = get_reciprocal(b); + const ulong k = mul_hi(as_uint2(a).s0, r) + ((ulong)(r) * as_uint2(a).s1) + a; + + const uint q = as_uint2(k).s1; + long tmp = a - ((ulong)(q) * b); + ((int*)&tmp)[1] -= (as_uint2(k).s1 < as_uint2(a).s1) ? b : 0; + + const int overshoot = ((int*)&tmp)[1] >> 31; + const int undershoot = as_int2(as_uint(b - 1) - tmp).s1 >> 31; + return (uint2)(q + overshoot - undershoot, as_uint2(tmp).s0 + (as_uint(overshoot) & b) - (as_uint(undershoot) & b)); +} + +inline uint fast_sqrt_v2(const ulong n1) +{ + float x = as_float((as_uint2(n1).s1 >> 9) + ((64U + 127U) << 23)); + + float x1 = native_rsqrt(x); + x = native_sqrt(x); + + // The following line does x1 *= 4294967296.0f; + x1 = as_float(as_uint(x1) + (32U << 23)); + + const uint x0 = as_uint(x) - (158U << 23); + const long delta0 = n1 - (as_ulong((uint2)(mul24(x0, x0), mul_hi(x0, x0))) << 18); + const float delta = convert_float_rte(as_int2(delta0).s1) * x1; + + uint result = (x0 << 10) + convert_int_rte(delta); + const uint s = result >> 1; + const uint b = result & 1; + + const ulong x2 = (ulong)(s) * (s + b) + ((ulong)(result) << 32) - n1; + if ((long)(x2 + as_int(b - 1)) >= 0) --result; + if ((long)(x2 + 0x100000000UL + s) < 0) ++result; + + return result; +} + +)===" diff --git a/src/backend/opencl/cl/cn/groestl256.cl b/src/backend/opencl/cl/cn/groestl256.cl new file mode 100644 index 00000000..1a7c96f0 --- /dev/null +++ b/src/backend/opencl/cl/cn/groestl256.cl @@ -0,0 +1,295 @@ +R"===( +/* $Id: groestl.c 260 2011-07-21 01:02:38Z tp $ */ +/* + * Groestl256 + * + * ==========================(LICENSE BEGIN)============================ + * Copyright (c) 2014 djm34 + * Copyright (c) 2007-2010 Projet RNRT SAPHIR + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * ===========================(LICENSE END)============================= + * + * @author Thomas Pornin + */ + +#define SPH_C64(x) x +#define SPH_ROTL64(x, y) rotate((x), (ulong)(y)) + + +#define C64e(x) ((SPH_C64(x) >> 56) \ + | ((SPH_C64(x) >> 40) & SPH_C64(0x000000000000FF00)) \ + | ((SPH_C64(x) >> 24) & SPH_C64(0x0000000000FF0000)) \ + | ((SPH_C64(x) >> 8) & SPH_C64(0x00000000FF000000)) \ + | ((SPH_C64(x) << 8) & SPH_C64(0x000000FF00000000)) \ + | ((SPH_C64(x) << 24) & SPH_C64(0x0000FF0000000000)) \ + | ((SPH_C64(x) << 40) & SPH_C64(0x00FF000000000000)) \ + | ((SPH_C64(x) << 56) & SPH_C64(0xFF00000000000000))) + +#define B64_0(x) ((x) & 0xFF) +#define B64_1(x) (((x) >> 8) & 0xFF) +#define B64_2(x) (((x) >> 16) & 0xFF) +#define B64_3(x) (((x) >> 24) & 0xFF) +#define B64_4(x) (((x) >> 32) & 0xFF) +#define B64_5(x) (((x) >> 40) & 0xFF) +#define B64_6(x) (((x) >> 48) & 0xFF) +#define B64_7(x) ((x) >> 56) +#define R64 SPH_ROTL64 +#define PC64(j, r) ((sph_u64)((j) + (r))) +#define QC64(j, r) (((sph_u64)(r) << 56) ^ (~((sph_u64)(j) << 56))) + +static const __constant ulong T0_G[] = +{ + 0xc6a597f4a5f432c6UL, 0xf884eb9784976ff8UL, 0xee99c7b099b05eeeUL, 0xf68df78c8d8c7af6UL, + 0xff0de5170d17e8ffUL, 0xd6bdb7dcbddc0ad6UL, 0xdeb1a7c8b1c816deUL, 0x915439fc54fc6d91UL, + 0x6050c0f050f09060UL, 0x0203040503050702UL, 0xcea987e0a9e02eceUL, 0x567dac877d87d156UL, + 0xe719d52b192bcce7UL, 0xb56271a662a613b5UL, 0x4de69a31e6317c4dUL, 0xec9ac3b59ab559ecUL, + 0x8f4505cf45cf408fUL, 0x1f9d3ebc9dbca31fUL, 0x894009c040c04989UL, 0xfa87ef92879268faUL, + 0xef15c53f153fd0efUL, 0xb2eb7f26eb2694b2UL, 0x8ec90740c940ce8eUL, 0xfb0bed1d0b1de6fbUL, + 0x41ec822fec2f6e41UL, 0xb3677da967a91ab3UL, 0x5ffdbe1cfd1c435fUL, 0x45ea8a25ea256045UL, + 0x23bf46dabfdaf923UL, 0x53f7a602f7025153UL, 0xe496d3a196a145e4UL, 0x9b5b2ded5bed769bUL, + 0x75c2ea5dc25d2875UL, 0xe11cd9241c24c5e1UL, 0x3dae7ae9aee9d43dUL, 0x4c6a98be6abef24cUL, + 0x6c5ad8ee5aee826cUL, 0x7e41fcc341c3bd7eUL, 0xf502f1060206f3f5UL, 0x834f1dd14fd15283UL, + 0x685cd0e45ce48c68UL, 0x51f4a207f4075651UL, 0xd134b95c345c8dd1UL, 0xf908e9180818e1f9UL, + 0xe293dfae93ae4ce2UL, 0xab734d9573953eabUL, 0x6253c4f553f59762UL, 0x2a3f54413f416b2aUL, + 0x080c10140c141c08UL, 0x955231f652f66395UL, 0x46658caf65afe946UL, 0x9d5e21e25ee27f9dUL, + 0x3028607828784830UL, 0x37a16ef8a1f8cf37UL, 0x0a0f14110f111b0aUL, 0x2fb55ec4b5c4eb2fUL, + 0x0e091c1b091b150eUL, 0x2436485a365a7e24UL, 0x1b9b36b69bb6ad1bUL, 0xdf3da5473d4798dfUL, + 0xcd26816a266aa7cdUL, 0x4e699cbb69bbf54eUL, 0x7fcdfe4ccd4c337fUL, 0xea9fcfba9fba50eaUL, + 0x121b242d1b2d3f12UL, 0x1d9e3ab99eb9a41dUL, 0x5874b09c749cc458UL, 0x342e68722e724634UL, + 0x362d6c772d774136UL, 0xdcb2a3cdb2cd11dcUL, 0xb4ee7329ee299db4UL, 0x5bfbb616fb164d5bUL, + 0xa4f65301f601a5a4UL, 0x764decd74dd7a176UL, 0xb76175a361a314b7UL, 0x7dcefa49ce49347dUL, + 0x527ba48d7b8ddf52UL, 0xdd3ea1423e429fddUL, 0x5e71bc937193cd5eUL, 0x139726a297a2b113UL, + 0xa6f55704f504a2a6UL, 0xb96869b868b801b9UL, 0x0000000000000000UL, 0xc12c99742c74b5c1UL, + 0x406080a060a0e040UL, 0xe31fdd211f21c2e3UL, 0x79c8f243c8433a79UL, 0xb6ed772ced2c9ab6UL, + 0xd4beb3d9bed90dd4UL, 0x8d4601ca46ca478dUL, 0x67d9ce70d9701767UL, 0x724be4dd4bddaf72UL, + 0x94de3379de79ed94UL, 0x98d42b67d467ff98UL, 0xb0e87b23e82393b0UL, 0x854a11de4ade5b85UL, + 0xbb6b6dbd6bbd06bbUL, 0xc52a917e2a7ebbc5UL, 0x4fe59e34e5347b4fUL, 0xed16c13a163ad7edUL, + 0x86c51754c554d286UL, 0x9ad72f62d762f89aUL, 0x6655ccff55ff9966UL, 0x119422a794a7b611UL, + 0x8acf0f4acf4ac08aUL, 0xe910c9301030d9e9UL, 0x0406080a060a0e04UL, 0xfe81e798819866feUL, + 0xa0f05b0bf00baba0UL, 0x7844f0cc44ccb478UL, 0x25ba4ad5bad5f025UL, 0x4be3963ee33e754bUL, + 0xa2f35f0ef30eaca2UL, 0x5dfeba19fe19445dUL, 0x80c01b5bc05bdb80UL, 0x058a0a858a858005UL, + 0x3fad7eecadecd33fUL, 0x21bc42dfbcdffe21UL, 0x7048e0d848d8a870UL, 0xf104f90c040cfdf1UL, + 0x63dfc67adf7a1963UL, 0x77c1ee58c1582f77UL, 0xaf75459f759f30afUL, 0x426384a563a5e742UL, + 0x2030405030507020UL, 0xe51ad12e1a2ecbe5UL, 0xfd0ee1120e12effdUL, 0xbf6d65b76db708bfUL, + 0x814c19d44cd45581UL, 0x1814303c143c2418UL, 0x26354c5f355f7926UL, 0xc32f9d712f71b2c3UL, + 0xbee16738e13886beUL, 0x35a26afda2fdc835UL, 0x88cc0b4fcc4fc788UL, 0x2e395c4b394b652eUL, + 0x93573df957f96a93UL, 0x55f2aa0df20d5855UL, 0xfc82e39d829d61fcUL, 0x7a47f4c947c9b37aUL, + 0xc8ac8befacef27c8UL, 0xbae76f32e73288baUL, 0x322b647d2b7d4f32UL, 0xe695d7a495a442e6UL, + 0xc0a09bfba0fb3bc0UL, 0x199832b398b3aa19UL, 0x9ed12768d168f69eUL, 0xa37f5d817f8122a3UL, + 0x446688aa66aaee44UL, 0x547ea8827e82d654UL, 0x3bab76e6abe6dd3bUL, 0x0b83169e839e950bUL, + 0x8cca0345ca45c98cUL, 0xc729957b297bbcc7UL, 0x6bd3d66ed36e056bUL, 0x283c50443c446c28UL, + 0xa779558b798b2ca7UL, 0xbce2633de23d81bcUL, 0x161d2c271d273116UL, 0xad76419a769a37adUL, + 0xdb3bad4d3b4d96dbUL, 0x6456c8fa56fa9e64UL, 0x744ee8d24ed2a674UL, 0x141e28221e223614UL, + 0x92db3f76db76e492UL, 0x0c0a181e0a1e120cUL, 0x486c90b46cb4fc48UL, 0xb8e46b37e4378fb8UL, + 0x9f5d25e75de7789fUL, 0xbd6e61b26eb20fbdUL, 0x43ef862aef2a6943UL, 0xc4a693f1a6f135c4UL, + 0x39a872e3a8e3da39UL, 0x31a462f7a4f7c631UL, 0xd337bd5937598ad3UL, 0xf28bff868b8674f2UL, + 0xd532b156325683d5UL, 0x8b430dc543c54e8bUL, 0x6e59dceb59eb856eUL, 0xdab7afc2b7c218daUL, + 0x018c028f8c8f8e01UL, 0xb16479ac64ac1db1UL, 0x9cd2236dd26df19cUL, 0x49e0923be03b7249UL, + 0xd8b4abc7b4c71fd8UL, 0xacfa4315fa15b9acUL, 0xf307fd090709faf3UL, 0xcf25856f256fa0cfUL, + 0xcaaf8feaafea20caUL, 0xf48ef3898e897df4UL, 0x47e98e20e9206747UL, 0x1018202818283810UL, + 0x6fd5de64d5640b6fUL, 0xf088fb83888373f0UL, 0x4a6f94b16fb1fb4aUL, 0x5c72b8967296ca5cUL, + 0x3824706c246c5438UL, 0x57f1ae08f1085f57UL, 0x73c7e652c7522173UL, 0x975135f351f36497UL, + 0xcb238d652365aecbUL, 0xa17c59847c8425a1UL, 0xe89ccbbf9cbf57e8UL, 0x3e217c6321635d3eUL, + 0x96dd377cdd7cea96UL, 0x61dcc27fdc7f1e61UL, 0x0d861a9186919c0dUL, 0x0f851e9485949b0fUL, + 0xe090dbab90ab4be0UL, 0x7c42f8c642c6ba7cUL, 0x71c4e257c4572671UL, 0xccaa83e5aae529ccUL, + 0x90d83b73d873e390UL, 0x06050c0f050f0906UL, 0xf701f5030103f4f7UL, 0x1c12383612362a1cUL, + 0xc2a39ffea3fe3cc2UL, 0x6a5fd4e15fe18b6aUL, 0xaef94710f910beaeUL, 0x69d0d26bd06b0269UL, + 0x17912ea891a8bf17UL, 0x995829e858e87199UL, 0x3a2774692769533aUL, 0x27b94ed0b9d0f727UL, + 0xd938a948384891d9UL, 0xeb13cd351335deebUL, 0x2bb356ceb3cee52bUL, 0x2233445533557722UL, + 0xd2bbbfd6bbd604d2UL, 0xa9704990709039a9UL, 0x07890e8089808707UL, 0x33a766f2a7f2c133UL, + 0x2db65ac1b6c1ec2dUL, 0x3c22786622665a3cUL, 0x15922aad92adb815UL, 0xc92089602060a9c9UL, + 0x874915db49db5c87UL, 0xaaff4f1aff1ab0aaUL, 0x5078a0887888d850UL, 0xa57a518e7a8e2ba5UL, + 0x038f068a8f8a8903UL, 0x59f8b213f8134a59UL, 0x0980129b809b9209UL, 0x1a1734391739231aUL, + 0x65daca75da751065UL, 0xd731b553315384d7UL, 0x84c61351c651d584UL, 0xd0b8bbd3b8d303d0UL, + 0x82c31f5ec35edc82UL, 0x29b052cbb0cbe229UL, 0x5a77b4997799c35aUL, 0x1e113c3311332d1eUL, + 0x7bcbf646cb463d7bUL, 0xa8fc4b1ffc1fb7a8UL, 0x6dd6da61d6610c6dUL, 0x2c3a584e3a4e622cUL +}; + +)===" +R"===( + +static const __constant ulong T4_G[] = +{ + 0xA5F432C6C6A597F4UL, 0x84976FF8F884EB97UL, 0x99B05EEEEE99C7B0UL, 0x8D8C7AF6F68DF78CUL, + 0x0D17E8FFFF0DE517UL, 0xBDDC0AD6D6BDB7DCUL, 0xB1C816DEDEB1A7C8UL, 0x54FC6D91915439FCUL, + 0x50F090606050C0F0UL, 0x0305070202030405UL, 0xA9E02ECECEA987E0UL, 0x7D87D156567DAC87UL, + 0x192BCCE7E719D52BUL, 0x62A613B5B56271A6UL, 0xE6317C4D4DE69A31UL, 0x9AB559ECEC9AC3B5UL, + 0x45CF408F8F4505CFUL, 0x9DBCA31F1F9D3EBCUL, 0x40C04989894009C0UL, 0x879268FAFA87EF92UL, + 0x153FD0EFEF15C53FUL, 0xEB2694B2B2EB7F26UL, 0xC940CE8E8EC90740UL, 0x0B1DE6FBFB0BED1DUL, + 0xEC2F6E4141EC822FUL, 0x67A91AB3B3677DA9UL, 0xFD1C435F5FFDBE1CUL, 0xEA25604545EA8A25UL, + 0xBFDAF92323BF46DAUL, 0xF702515353F7A602UL, 0x96A145E4E496D3A1UL, 0x5BED769B9B5B2DEDUL, + 0xC25D287575C2EA5DUL, 0x1C24C5E1E11CD924UL, 0xAEE9D43D3DAE7AE9UL, 0x6ABEF24C4C6A98BEUL, + 0x5AEE826C6C5AD8EEUL, 0x41C3BD7E7E41FCC3UL, 0x0206F3F5F502F106UL, 0x4FD15283834F1DD1UL, + 0x5CE48C68685CD0E4UL, 0xF407565151F4A207UL, 0x345C8DD1D134B95CUL, 0x0818E1F9F908E918UL, + 0x93AE4CE2E293DFAEUL, 0x73953EABAB734D95UL, 0x53F597626253C4F5UL, 0x3F416B2A2A3F5441UL, + 0x0C141C08080C1014UL, 0x52F66395955231F6UL, 0x65AFE94646658CAFUL, 0x5EE27F9D9D5E21E2UL, + 0x2878483030286078UL, 0xA1F8CF3737A16EF8UL, 0x0F111B0A0A0F1411UL, 0xB5C4EB2F2FB55EC4UL, + 0x091B150E0E091C1BUL, 0x365A7E242436485AUL, 0x9BB6AD1B1B9B36B6UL, 0x3D4798DFDF3DA547UL, + 0x266AA7CDCD26816AUL, 0x69BBF54E4E699CBBUL, 0xCD4C337F7FCDFE4CUL, 0x9FBA50EAEA9FCFBAUL, + 0x1B2D3F12121B242DUL, 0x9EB9A41D1D9E3AB9UL, 0x749CC4585874B09CUL, 0x2E724634342E6872UL, + 0x2D774136362D6C77UL, 0xB2CD11DCDCB2A3CDUL, 0xEE299DB4B4EE7329UL, 0xFB164D5B5BFBB616UL, + 0xF601A5A4A4F65301UL, 0x4DD7A176764DECD7UL, 0x61A314B7B76175A3UL, 0xCE49347D7DCEFA49UL, + 0x7B8DDF52527BA48DUL, 0x3E429FDDDD3EA142UL, 0x7193CD5E5E71BC93UL, 0x97A2B113139726A2UL, + 0xF504A2A6A6F55704UL, 0x68B801B9B96869B8UL, 0x0000000000000000UL, 0x2C74B5C1C12C9974UL, + 0x60A0E040406080A0UL, 0x1F21C2E3E31FDD21UL, 0xC8433A7979C8F243UL, 0xED2C9AB6B6ED772CUL, + 0xBED90DD4D4BEB3D9UL, 0x46CA478D8D4601CAUL, 0xD970176767D9CE70UL, 0x4BDDAF72724BE4DDUL, + 0xDE79ED9494DE3379UL, 0xD467FF9898D42B67UL, 0xE82393B0B0E87B23UL, 0x4ADE5B85854A11DEUL, + 0x6BBD06BBBB6B6DBDUL, 0x2A7EBBC5C52A917EUL, 0xE5347B4F4FE59E34UL, 0x163AD7EDED16C13AUL, + 0xC554D28686C51754UL, 0xD762F89A9AD72F62UL, 0x55FF99666655CCFFUL, 0x94A7B611119422A7UL, + 0xCF4AC08A8ACF0F4AUL, 0x1030D9E9E910C930UL, 0x060A0E040406080AUL, 0x819866FEFE81E798UL, + 0xF00BABA0A0F05B0BUL, 0x44CCB4787844F0CCUL, 0xBAD5F02525BA4AD5UL, 0xE33E754B4BE3963EUL, + 0xF30EACA2A2F35F0EUL, 0xFE19445D5DFEBA19UL, 0xC05BDB8080C01B5BUL, 0x8A858005058A0A85UL, + 0xADECD33F3FAD7EECUL, 0xBCDFFE2121BC42DFUL, 0x48D8A8707048E0D8UL, 0x040CFDF1F104F90CUL, + 0xDF7A196363DFC67AUL, 0xC1582F7777C1EE58UL, 0x759F30AFAF75459FUL, 0x63A5E742426384A5UL, + 0x3050702020304050UL, 0x1A2ECBE5E51AD12EUL, 0x0E12EFFDFD0EE112UL, 0x6DB708BFBF6D65B7UL, + 0x4CD45581814C19D4UL, 0x143C24181814303CUL, 0x355F792626354C5FUL, 0x2F71B2C3C32F9D71UL, + 0xE13886BEBEE16738UL, 0xA2FDC83535A26AFDUL, 0xCC4FC78888CC0B4FUL, 0x394B652E2E395C4BUL, + 0x57F96A9393573DF9UL, 0xF20D585555F2AA0DUL, 0x829D61FCFC82E39DUL, 0x47C9B37A7A47F4C9UL, + 0xACEF27C8C8AC8BEFUL, 0xE73288BABAE76F32UL, 0x2B7D4F32322B647DUL, 0x95A442E6E695D7A4UL, + 0xA0FB3BC0C0A09BFBUL, 0x98B3AA19199832B3UL, 0xD168F69E9ED12768UL, 0x7F8122A3A37F5D81UL, + 0x66AAEE44446688AAUL, 0x7E82D654547EA882UL, 0xABE6DD3B3BAB76E6UL, 0x839E950B0B83169EUL, + 0xCA45C98C8CCA0345UL, 0x297BBCC7C729957BUL, 0xD36E056B6BD3D66EUL, 0x3C446C28283C5044UL, + 0x798B2CA7A779558BUL, 0xE23D81BCBCE2633DUL, 0x1D273116161D2C27UL, 0x769A37ADAD76419AUL, + 0x3B4D96DBDB3BAD4DUL, 0x56FA9E646456C8FAUL, 0x4ED2A674744EE8D2UL, 0x1E223614141E2822UL, + 0xDB76E49292DB3F76UL, 0x0A1E120C0C0A181EUL, 0x6CB4FC48486C90B4UL, 0xE4378FB8B8E46B37UL, + 0x5DE7789F9F5D25E7UL, 0x6EB20FBDBD6E61B2UL, 0xEF2A694343EF862AUL, 0xA6F135C4C4A693F1UL, + 0xA8E3DA3939A872E3UL, 0xA4F7C63131A462F7UL, 0x37598AD3D337BD59UL, 0x8B8674F2F28BFF86UL, + 0x325683D5D532B156UL, 0x43C54E8B8B430DC5UL, 0x59EB856E6E59DCEBUL, 0xB7C218DADAB7AFC2UL, + 0x8C8F8E01018C028FUL, 0x64AC1DB1B16479ACUL, 0xD26DF19C9CD2236DUL, 0xE03B724949E0923BUL, + 0xB4C71FD8D8B4ABC7UL, 0xFA15B9ACACFA4315UL, 0x0709FAF3F307FD09UL, 0x256FA0CFCF25856FUL, + 0xAFEA20CACAAF8FEAUL, 0x8E897DF4F48EF389UL, 0xE920674747E98E20UL, 0x1828381010182028UL, + 0xD5640B6F6FD5DE64UL, 0x888373F0F088FB83UL, 0x6FB1FB4A4A6F94B1UL, 0x7296CA5C5C72B896UL, + 0x246C54383824706CUL, 0xF1085F5757F1AE08UL, 0xC752217373C7E652UL, 0x51F36497975135F3UL, + 0x2365AECBCB238D65UL, 0x7C8425A1A17C5984UL, 0x9CBF57E8E89CCBBFUL, 0x21635D3E3E217C63UL, + 0xDD7CEA9696DD377CUL, 0xDC7F1E6161DCC27FUL, 0x86919C0D0D861A91UL, 0x85949B0F0F851E94UL, + 0x90AB4BE0E090DBABUL, 0x42C6BA7C7C42F8C6UL, 0xC457267171C4E257UL, 0xAAE529CCCCAA83E5UL, + 0xD873E39090D83B73UL, 0x050F090606050C0FUL, 0x0103F4F7F701F503UL, 0x12362A1C1C123836UL, + 0xA3FE3CC2C2A39FFEUL, 0x5FE18B6A6A5FD4E1UL, 0xF910BEAEAEF94710UL, 0xD06B026969D0D26BUL, + 0x91A8BF1717912EA8UL, 0x58E87199995829E8UL, 0x2769533A3A277469UL, 0xB9D0F72727B94ED0UL, + 0x384891D9D938A948UL, 0x1335DEEBEB13CD35UL, 0xB3CEE52B2BB356CEUL, 0x3355772222334455UL, + 0xBBD604D2D2BBBFD6UL, 0x709039A9A9704990UL, 0x8980870707890E80UL, 0xA7F2C13333A766F2UL, + 0xB6C1EC2D2DB65AC1UL, 0x22665A3C3C227866UL, 0x92ADB81515922AADUL, 0x2060A9C9C9208960UL, + 0x49DB5C87874915DBUL, 0xFF1AB0AAAAFF4F1AUL, 0x7888D8505078A088UL, 0x7A8E2BA5A57A518EUL, + 0x8F8A8903038F068AUL, 0xF8134A5959F8B213UL, 0x809B92090980129BUL, 0x1739231A1A173439UL, + 0xDA75106565DACA75UL, 0x315384D7D731B553UL, 0xC651D58484C61351UL, 0xB8D303D0D0B8BBD3UL, + 0xC35EDC8282C31F5EUL, 0xB0CBE22929B052CBUL, 0x7799C35A5A77B499UL, 0x11332D1E1E113C33UL, + 0xCB463D7B7BCBF646UL, 0xFC1FB7A8A8FC4B1FUL, 0xD6610C6D6DD6DA61UL, 0x3A4E622C2C3A584EUL +}; + +#define RSTT(d, a, b0, b1, b2, b3, b4, b5, b6, b7) do { \ + t[d] = T0_G[B64_0(a[b0])] \ + ^ R64(T0_G[B64_1(a[b1])], 8) \ + ^ R64(T0_G[B64_2(a[b2])], 16) \ + ^ R64(T0_G[B64_3(a[b3])], 24) \ + ^ T4_G[B64_4(a[b4])] \ + ^ R64(T4_G[B64_5(a[b5])], 8) \ + ^ R64(T4_G[B64_6(a[b6])], 16) \ + ^ R64(T4_G[B64_7(a[b7])], 24); \ + } while (0) + +#define ROUND_SMALL_P(a, r) do { \ + ulong t[8]; \ + a[0] ^= PC64(0x00, r); \ + a[1] ^= PC64(0x10, r); \ + a[2] ^= PC64(0x20, r); \ + a[3] ^= PC64(0x30, r); \ + a[4] ^= PC64(0x40, r); \ + a[5] ^= PC64(0x50, r); \ + a[6] ^= PC64(0x60, r); \ + a[7] ^= PC64(0x70, r); \ + RSTT(0, a, 0, 1, 2, 3, 4, 5, 6, 7); \ + RSTT(1, a, 1, 2, 3, 4, 5, 6, 7, 0); \ + RSTT(2, a, 2, 3, 4, 5, 6, 7, 0, 1); \ + RSTT(3, a, 3, 4, 5, 6, 7, 0, 1, 2); \ + RSTT(4, a, 4, 5, 6, 7, 0, 1, 2, 3); \ + RSTT(5, a, 5, 6, 7, 0, 1, 2, 3, 4); \ + RSTT(6, a, 6, 7, 0, 1, 2, 3, 4, 5); \ + RSTT(7, a, 7, 0, 1, 2, 3, 4, 5, 6); \ + a[0] = t[0]; \ + a[1] = t[1]; \ + a[2] = t[2]; \ + a[3] = t[3]; \ + a[4] = t[4]; \ + a[5] = t[5]; \ + a[6] = t[6]; \ + a[7] = t[7]; \ + } while (0) + +#define ROUND_SMALL_Pf(a,r) do { \ + a[0] ^= PC64(0x00, r); \ + a[1] ^= PC64(0x10, r); \ + a[2] ^= PC64(0x20, r); \ + a[3] ^= PC64(0x30, r); \ + a[4] ^= PC64(0x40, r); \ + a[5] ^= PC64(0x50, r); \ + a[6] ^= PC64(0x60, r); \ + a[7] ^= PC64(0x70, r); \ + RSTT(7, a, 7, 0, 1, 2, 3, 4, 5, 6); \ + a[7] = t[7]; \ + } while (0) + +#define ROUND_SMALL_Q(a, r) do { \ + ulong t[8]; \ + a[0] ^= QC64(0x00, r); \ + a[1] ^= QC64(0x10, r); \ + a[2] ^= QC64(0x20, r); \ + a[3] ^= QC64(0x30, r); \ + a[4] ^= QC64(0x40, r); \ + a[5] ^= QC64(0x50, r); \ + a[6] ^= QC64(0x60, r); \ + a[7] ^= QC64(0x70, r); \ + RSTT(0, a, 1, 3, 5, 7, 0, 2, 4, 6); \ + RSTT(1, a, 2, 4, 6, 0, 1, 3, 5, 7); \ + RSTT(2, a, 3, 5, 7, 1, 2, 4, 6, 0); \ + RSTT(3, a, 4, 6, 0, 2, 3, 5, 7, 1); \ + RSTT(4, a, 5, 7, 1, 3, 4, 6, 0, 2); \ + RSTT(5, a, 6, 0, 2, 4, 5, 7, 1, 3); \ + RSTT(6, a, 7, 1, 3, 5, 6, 0, 2, 4); \ + RSTT(7, a, 0, 2, 4, 6, 7, 1, 3, 5); \ + a[0] = t[0]; \ + a[1] = t[1]; \ + a[2] = t[2]; \ + a[3] = t[3]; \ + a[4] = t[4]; \ + a[5] = t[5]; \ + a[6] = t[6]; \ + a[7] = t[7]; \ + } while (0) + +#define PERM_SMALL_P(a) do { \ + for (int r = 0; r < 10; r ++) \ + ROUND_SMALL_P(a, r); \ + } while (0) + +#define PERM_SMALL_Pf(a) do { \ + for (int r = 0; r < 9; r ++) { \ + ROUND_SMALL_P(a, r);} \ + ROUND_SMALL_Pf(a,9); \ + } while (0) + +#define PERM_SMALL_Q(a) do { \ + for (int r = 0; r < 10; r ++) \ + ROUND_SMALL_Q(a, r); \ + } while (0) + +)===" + \ No newline at end of file diff --git a/src/backend/opencl/cl/cn/jh.cl b/src/backend/opencl/cl/cn/jh.cl new file mode 100644 index 00000000..fe70ea30 --- /dev/null +++ b/src/backend/opencl/cl/cn/jh.cl @@ -0,0 +1,274 @@ +R"===( +/* $Id: jh.c 255 2011-06-07 19:50:20Z tp $ */ +/* + * JH implementation. + * + * ==========================(LICENSE BEGIN)============================ + * + * Copyright (c) 2007-2010 Projet RNRT SAPHIR + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + * ===========================(LICENSE END)============================= + * + * @author Thomas Pornin + */ + +#define SPH_JH_64 1 +#define SPH_LITTLE_ENDIAN 1 + +#define SPH_C32(x) x +#define SPH_C64(x) x +typedef uint sph_u32; +typedef ulong sph_u64; + +/* + * The internal bitslice representation may use either big-endian or + * little-endian (true bitslice operations do not care about the bit + * ordering, and the bit-swapping linear operations in JH happen to + * be invariant through endianness-swapping). The constants must be + * defined according to the chosen endianness; we use some + * byte-swapping macros for that. + */ + +#if SPH_LITTLE_ENDIAN + +#define C32e(x) ((SPH_C32(x) >> 24) \ + | ((SPH_C32(x) >> 8) & SPH_C32(0x0000FF00)) \ + | ((SPH_C32(x) << 8) & SPH_C32(0x00FF0000)) \ + | ((SPH_C32(x) << 24) & SPH_C32(0xFF000000))) +#define dec32e_aligned sph_dec32le_aligned +#define enc32e sph_enc32le + +#define C64e(x) ((SPH_C64(x) >> 56) \ + | ((SPH_C64(x) >> 40) & SPH_C64(0x000000000000FF00)) \ + | ((SPH_C64(x) >> 24) & SPH_C64(0x0000000000FF0000)) \ + | ((SPH_C64(x) >> 8) & SPH_C64(0x00000000FF000000)) \ + | ((SPH_C64(x) << 8) & SPH_C64(0x000000FF00000000)) \ + | ((SPH_C64(x) << 24) & SPH_C64(0x0000FF0000000000)) \ + | ((SPH_C64(x) << 40) & SPH_C64(0x00FF000000000000)) \ + | ((SPH_C64(x) << 56) & SPH_C64(0xFF00000000000000))) +#define dec64e_aligned sph_dec64le_aligned +#define enc64e sph_enc64le + +#else + +#define C32e(x) SPH_C32(x) +#define dec32e_aligned sph_dec32be_aligned +#define enc32e sph_enc32be +#define C64e(x) SPH_C64(x) +#define dec64e_aligned sph_dec64be_aligned +#define enc64e sph_enc64be + +#endif + +#define Sb(x0, x1, x2, x3, c) do { \ + x3 = ~x3; \ + x0 ^= (c) & ~x2; \ + tmp = (c) ^ (x0 & x1); \ + x0 ^= x2 & x3; \ + x3 ^= ~x1 & x2; \ + x1 ^= x0 & x2; \ + x2 ^= x0 & ~x3; \ + x0 ^= x1 | x3; \ + x3 ^= x1 & x2; \ + x1 ^= tmp & x0; \ + x2 ^= tmp; \ + } while (0) + +#define Lb(x0, x1, x2, x3, x4, x5, x6, x7) do { \ + x4 ^= x1; \ + x5 ^= x2; \ + x6 ^= x3 ^ x0; \ + x7 ^= x0; \ + x0 ^= x5; \ + x1 ^= x6; \ + x2 ^= x7 ^ x4; \ + x3 ^= x4; \ + } while (0) + +static const __constant ulong C[] = +{ + 0x67F815DFA2DED572UL, 0x571523B70A15847BUL, 0xF6875A4D90D6AB81UL, 0x402BD1C3C54F9F4EUL, + 0x9CFA455CE03A98EAUL, 0x9A99B26699D2C503UL, 0x8A53BBF2B4960266UL, 0x31A2DB881A1456B5UL, + 0xDB0E199A5C5AA303UL, 0x1044C1870AB23F40UL, 0x1D959E848019051CUL, 0xDCCDE75EADEB336FUL, + 0x416BBF029213BA10UL, 0xD027BBF7156578DCUL, 0x5078AA3739812C0AUL, 0xD3910041D2BF1A3FUL, + 0x907ECCF60D5A2D42UL, 0xCE97C0929C9F62DDUL, 0xAC442BC70BA75C18UL, 0x23FCC663D665DFD1UL, + 0x1AB8E09E036C6E97UL, 0xA8EC6C447E450521UL, 0xFA618E5DBB03F1EEUL, 0x97818394B29796FDUL, + 0x2F3003DB37858E4AUL, 0x956A9FFB2D8D672AUL, 0x6C69B8F88173FE8AUL, 0x14427FC04672C78AUL, + 0xC45EC7BD8F15F4C5UL, 0x80BB118FA76F4475UL, 0xBC88E4AEB775DE52UL, 0xF4A3A6981E00B882UL, + 0x1563A3A9338FF48EUL, 0x89F9B7D524565FAAUL, 0xFDE05A7C20EDF1B6UL, 0x362C42065AE9CA36UL, + 0x3D98FE4E433529CEUL, 0xA74B9A7374F93A53UL, 0x86814E6F591FF5D0UL, 0x9F5AD8AF81AD9D0EUL, + 0x6A6234EE670605A7UL, 0x2717B96EBE280B8BUL, 0x3F1080C626077447UL, 0x7B487EC66F7EA0E0UL, + 0xC0A4F84AA50A550DUL, 0x9EF18E979FE7E391UL, 0xD48D605081727686UL, 0x62B0E5F3415A9E7EUL, + 0x7A205440EC1F9FFCUL, 0x84C9F4CE001AE4E3UL, 0xD895FA9DF594D74FUL, 0xA554C324117E2E55UL, + 0x286EFEBD2872DF5BUL, 0xB2C4A50FE27FF578UL, 0x2ED349EEEF7C8905UL, 0x7F5928EB85937E44UL, + 0x4A3124B337695F70UL, 0x65E4D61DF128865EUL, 0xE720B95104771BC7UL, 0x8A87D423E843FE74UL, + 0xF2947692A3E8297DUL, 0xC1D9309B097ACBDDUL, 0xE01BDC5BFB301B1DUL, 0xBF829CF24F4924DAUL, + 0xFFBF70B431BAE7A4UL, 0x48BCF8DE0544320DUL, 0x39D3BB5332FCAE3BUL, 0xA08B29E0C1C39F45UL, + 0x0F09AEF7FD05C9E5UL, 0x34F1904212347094UL, 0x95ED44E301B771A2UL, 0x4A982F4F368E3BE9UL, + 0x15F66CA0631D4088UL, 0xFFAF52874B44C147UL, 0x30C60AE2F14ABB7EUL, 0xE68C6ECCC5B67046UL, + 0x00CA4FBD56A4D5A4UL, 0xAE183EC84B849DDAUL, 0xADD1643045CE5773UL, 0x67255C1468CEA6E8UL, + 0x16E10ECBF28CDAA3UL, 0x9A99949A5806E933UL, 0x7B846FC220B2601FUL, 0x1885D1A07FACCED1UL, + 0xD319DD8DA15B5932UL, 0x46B4A5AAC01C9A50UL, 0xBA6B04E467633D9FUL, 0x7EEE560BAB19CAF6UL, + 0x742128A9EA79B11FUL, 0xEE51363B35F7BDE9UL, 0x76D350755AAC571DUL, 0x01707DA3FEC2463AUL, + 0x42D8A498AFC135F7UL, 0x79676B9E20ECED78UL, 0xA8DB3AEA15638341UL, 0x832C83324D3BC3FAUL, + 0xF347271C1F3B40A7UL, 0x9A762DB734F04059UL, 0xFD4F21D26C4E3EE7UL, 0xEF5957DC398DFDB8UL, + 0xDAEB492B490C9B8DUL, 0x0D70F36849D7A25BUL, 0x84558D7AD0AE3B7DUL, 0x658EF8E4F0E9A5F5UL, + 0x533B1036F4A2B8A0UL, 0x5AEC3E759E07A80CUL, 0x4F88E85692946891UL, 0x4CBCBAF8555CB05BUL, + 0x7B9487F3993BBBE3UL, 0x5D1C6B72D6F4DA75UL, 0x6DB334DC28ACAE64UL, 0x71DB28B850A5346CUL, + 0x2A518D10F2E261F8UL, 0xFC75DD593364DBE3UL, 0xA23FCE43F1BCAC1CUL, 0xB043E8023CD1BB67UL, + 0x75A12988CA5B0A33UL, 0x5C5316B44D19347FUL, 0x1E4D790EC3943B92UL, 0x3FAFEEB6D7757479UL, + 0x21391ABEF7D4A8EAUL, 0x5127234C097EF45CUL, 0xD23C32BA5324A326UL, 0xADD5A66D4A17A344UL, + 0x08C9F2AFA63E1DB5UL, 0x563C6B91983D5983UL, 0x4D608672A17CF84CUL, 0xF6C76E08CC3EE246UL, + 0x5E76BCB1B333982FUL, 0x2AE6C4EFA566D62BUL, 0x36D4C1BEE8B6F406UL, 0x6321EFBC1582EE74UL, + 0x69C953F40D4EC1FDUL, 0x26585806C45A7DA7UL, 0x16FAE0061614C17EUL, 0x3F9D63283DAF907EUL, + 0x0CD29B00E3F2C9D2UL, 0x300CD4B730CEAA5FUL, 0x9832E0F216512A74UL, 0x9AF8CEE3D830EB0DUL, + 0x9279F1B57B9EC54BUL, 0xD36886046EE651FFUL, 0x316796E6574D239BUL, 0x05750A17F3A6E6CCUL, + 0xCE6C3213D98176B1UL, 0x62A205F88452173CUL, 0x47154778B3CB2BF4UL, 0x486A9323825446FFUL, + 0x65655E4E0758DF38UL, 0x8E5086FC897CFCF2UL, 0x86CA0BD0442E7031UL, 0x4E477830A20940F0UL, + 0x8338F7D139EEA065UL, 0xBD3A2CE437E95EF7UL, 0x6FF8130126B29721UL, 0xE7DE9FEFD1ED44A3UL, + 0xD992257615DFA08BUL, 0xBE42DC12F6F7853CUL, 0x7EB027AB7CECA7D8UL, 0xDEA83EAADA7D8D53UL, + 0xD86902BD93CE25AAUL, 0xF908731AFD43F65AUL, 0xA5194A17DAEF5FC0UL, 0x6A21FD4C33664D97UL, + 0x701541DB3198B435UL, 0x9B54CDEDBB0F1EEAUL, 0x72409751A163D09AUL, 0xE26F4791BF9D75F6UL +}; + +#define Ceven_hi(r) (C[((r) << 2) + 0]) +#define Ceven_lo(r) (C[((r) << 2) + 1]) +#define Codd_hi(r) (C[((r) << 2) + 2]) +#define Codd_lo(r) (C[((r) << 2) + 3]) + +#define S(x0, x1, x2, x3, cb, r) do { \ + Sb(x0 ## h, x1 ## h, x2 ## h, x3 ## h, cb ## hi(r)); \ + Sb(x0 ## l, x1 ## l, x2 ## l, x3 ## l, cb ## lo(r)); \ + } while (0) + +#define L(x0, x1, x2, x3, x4, x5, x6, x7) do { \ + Lb(x0 ## h, x1 ## h, x2 ## h, x3 ## h, \ + x4 ## h, x5 ## h, x6 ## h, x7 ## h); \ + Lb(x0 ## l, x1 ## l, x2 ## l, x3 ## l, \ + x4 ## l, x5 ## l, x6 ## l, x7 ## l); \ + } while (0) + +#define Wz(x, c, n) do { \ + sph_u64 t = (x ## h & (c)) << (n); \ + x ## h = ((x ## h >> (n)) & (c)) | t; \ + t = (x ## l & (c)) << (n); \ + x ## l = ((x ## l >> (n)) & (c)) | t; \ + } while (0) + +#define W0(x) Wz(x, SPH_C64(0x5555555555555555), 1) +#define W1(x) Wz(x, SPH_C64(0x3333333333333333), 2) +#define W2(x) Wz(x, SPH_C64(0x0F0F0F0F0F0F0F0F), 4) +#define W3(x) Wz(x, SPH_C64(0x00FF00FF00FF00FF), 8) +#define W4(x) Wz(x, SPH_C64(0x0000FFFF0000FFFF), 16) +#define W5(x) Wz(x, SPH_C64(0x00000000FFFFFFFF), 32) +#define W6(x) do { \ + sph_u64 t = x ## h; \ + x ## h = x ## l; \ + x ## l = t; \ + } while (0) + +#define SL(ro) SLu(r + ro, ro) + +#define SLu(r, ro) do { \ + S(h0, h2, h4, h6, Ceven_, r); \ + S(h1, h3, h5, h7, Codd_, r); \ + L(h0, h2, h4, h6, h1, h3, h5, h7); \ + W ## ro(h1); \ + W ## ro(h3); \ + W ## ro(h5); \ + W ## ro(h7); \ + } while (0) + +#if SPH_SMALL_FOOTPRINT_JH + +/* + * The "small footprint" 64-bit version just uses a partially unrolled + * loop. + */ + +#define E8 do { \ + unsigned r; \ + for (r = 0; r < 42; r += 7) { \ + SL(0); \ + SL(1); \ + SL(2); \ + SL(3); \ + SL(4); \ + SL(5); \ + SL(6); \ + } \ + } while (0) + +#else + +/* + * On a "true 64-bit" architecture, we can unroll at will. + */ + +#define E8 do { \ + SLu( 0, 0); \ + SLu( 1, 1); \ + SLu( 2, 2); \ + SLu( 3, 3); \ + SLu( 4, 4); \ + SLu( 5, 5); \ + SLu( 6, 6); \ + SLu( 7, 0); \ + SLu( 8, 1); \ + SLu( 9, 2); \ + SLu(10, 3); \ + SLu(11, 4); \ + SLu(12, 5); \ + SLu(13, 6); \ + SLu(14, 0); \ + SLu(15, 1); \ + SLu(16, 2); \ + SLu(17, 3); \ + SLu(18, 4); \ + SLu(19, 5); \ + SLu(20, 6); \ + SLu(21, 0); \ + SLu(22, 1); \ + SLu(23, 2); \ + SLu(24, 3); \ + SLu(25, 4); \ + SLu(26, 5); \ + SLu(27, 6); \ + SLu(28, 0); \ + SLu(29, 1); \ + SLu(30, 2); \ + SLu(31, 3); \ + SLu(32, 4); \ + SLu(33, 5); \ + SLu(34, 6); \ + SLu(35, 0); \ + SLu(36, 1); \ + SLu(37, 2); \ + SLu(38, 3); \ + SLu(39, 4); \ + SLu(40, 5); \ + SLu(41, 6); \ + } while (0) + +#endif + +)===" diff --git a/src/backend/opencl/cl/cn/wolf-aes.cl b/src/backend/opencl/cl/cn/wolf-aes.cl new file mode 100644 index 00000000..aa9839d5 --- /dev/null +++ b/src/backend/opencl/cl/cn/wolf-aes.cl @@ -0,0 +1,153 @@ +R"===( +#ifndef WOLF_AES_CL +#define WOLF_AES_CL + +#ifdef cl_amd_media_ops2 +# pragma OPENCL EXTENSION cl_amd_media_ops2 : enable + +# define xmrig_amd_bfe(src0, src1, src2) amd_bfe(src0, src1, src2) +#else +/* taken from: https://www.khronos.org/registry/OpenCL/extensions/amd/cl_amd_media_ops2.txt + * Built-in Function: + * uintn amd_bfe (uintn src0, uintn src1, uintn src2) + * Description + * NOTE: operator >> below represent logical right shift + * offset = src1.s0 & 31; + * width = src2.s0 & 31; + * if width = 0 + * dst.s0 = 0; + * else if (offset + width) < 32 + * dst.s0 = (src0.s0 << (32 - offset - width)) >> (32 - width); + * else + * dst.s0 = src0.s0 >> offset; + * similar operation applied to other components of the vectors + */ +inline int xmrig_amd_bfe(const uint src0, const uint offset, const uint width) +{ + /* casts are removed because we can implement everything as uint + * int offset = src1; + * int width = src2; + * remove check for edge case, this function is always called with + * `width==8` + * @code + * if ( width == 0 ) + * return 0; + * @endcode + */ + if ((offset + width) < 32u) { + return (src0 << (32u - offset - width)) >> (32u - width); + } + + return src0 >> offset; +} +#endif + + +// AES table - the other three are generated on the fly + +static const __constant uint AES0_C[256] = +{ + 0xA56363C6U, 0x847C7CF8U, 0x997777EEU, 0x8D7B7BF6U, + 0x0DF2F2FFU, 0xBD6B6BD6U, 0xB16F6FDEU, 0x54C5C591U, + 0x50303060U, 0x03010102U, 0xA96767CEU, 0x7D2B2B56U, + 0x19FEFEE7U, 0x62D7D7B5U, 0xE6ABAB4DU, 0x9A7676ECU, + 0x45CACA8FU, 0x9D82821FU, 0x40C9C989U, 0x877D7DFAU, + 0x15FAFAEFU, 0xEB5959B2U, 0xC947478EU, 0x0BF0F0FBU, + 0xECADAD41U, 0x67D4D4B3U, 0xFDA2A25FU, 0xEAAFAF45U, + 0xBF9C9C23U, 0xF7A4A453U, 0x967272E4U, 0x5BC0C09BU, + 0xC2B7B775U, 0x1CFDFDE1U, 0xAE93933DU, 0x6A26264CU, + 0x5A36366CU, 0x413F3F7EU, 0x02F7F7F5U, 0x4FCCCC83U, + 0x5C343468U, 0xF4A5A551U, 0x34E5E5D1U, 0x08F1F1F9U, + 0x937171E2U, 0x73D8D8ABU, 0x53313162U, 0x3F15152AU, + 0x0C040408U, 0x52C7C795U, 0x65232346U, 0x5EC3C39DU, + 0x28181830U, 0xA1969637U, 0x0F05050AU, 0xB59A9A2FU, + 0x0907070EU, 0x36121224U, 0x9B80801BU, 0x3DE2E2DFU, + 0x26EBEBCDU, 0x6927274EU, 0xCDB2B27FU, 0x9F7575EAU, + 0x1B090912U, 0x9E83831DU, 0x742C2C58U, 0x2E1A1A34U, + 0x2D1B1B36U, 0xB26E6EDCU, 0xEE5A5AB4U, 0xFBA0A05BU, + 0xF65252A4U, 0x4D3B3B76U, 0x61D6D6B7U, 0xCEB3B37DU, + 0x7B292952U, 0x3EE3E3DDU, 0x712F2F5EU, 0x97848413U, + 0xF55353A6U, 0x68D1D1B9U, 0x00000000U, 0x2CEDEDC1U, + 0x60202040U, 0x1FFCFCE3U, 0xC8B1B179U, 0xED5B5BB6U, + 0xBE6A6AD4U, 0x46CBCB8DU, 0xD9BEBE67U, 0x4B393972U, + 0xDE4A4A94U, 0xD44C4C98U, 0xE85858B0U, 0x4ACFCF85U, + 0x6BD0D0BBU, 0x2AEFEFC5U, 0xE5AAAA4FU, 0x16FBFBEDU, + 0xC5434386U, 0xD74D4D9AU, 0x55333366U, 0x94858511U, + 0xCF45458AU, 0x10F9F9E9U, 0x06020204U, 0x817F7FFEU, + 0xF05050A0U, 0x443C3C78U, 0xBA9F9F25U, 0xE3A8A84BU, + 0xF35151A2U, 0xFEA3A35DU, 0xC0404080U, 0x8A8F8F05U, + 0xAD92923FU, 0xBC9D9D21U, 0x48383870U, 0x04F5F5F1U, + 0xDFBCBC63U, 0xC1B6B677U, 0x75DADAAFU, 0x63212142U, + 0x30101020U, 0x1AFFFFE5U, 0x0EF3F3FDU, 0x6DD2D2BFU, + 0x4CCDCD81U, 0x140C0C18U, 0x35131326U, 0x2FECECC3U, + 0xE15F5FBEU, 0xA2979735U, 0xCC444488U, 0x3917172EU, + 0x57C4C493U, 0xF2A7A755U, 0x827E7EFCU, 0x473D3D7AU, + 0xAC6464C8U, 0xE75D5DBAU, 0x2B191932U, 0x957373E6U, + 0xA06060C0U, 0x98818119U, 0xD14F4F9EU, 0x7FDCDCA3U, + 0x66222244U, 0x7E2A2A54U, 0xAB90903BU, 0x8388880BU, + 0xCA46468CU, 0x29EEEEC7U, 0xD3B8B86BU, 0x3C141428U, + 0x79DEDEA7U, 0xE25E5EBCU, 0x1D0B0B16U, 0x76DBDBADU, + 0x3BE0E0DBU, 0x56323264U, 0x4E3A3A74U, 0x1E0A0A14U, + 0xDB494992U, 0x0A06060CU, 0x6C242448U, 0xE45C5CB8U, + 0x5DC2C29FU, 0x6ED3D3BDU, 0xEFACAC43U, 0xA66262C4U, + 0xA8919139U, 0xA4959531U, 0x37E4E4D3U, 0x8B7979F2U, + 0x32E7E7D5U, 0x43C8C88BU, 0x5937376EU, 0xB76D6DDAU, + 0x8C8D8D01U, 0x64D5D5B1U, 0xD24E4E9CU, 0xE0A9A949U, + 0xB46C6CD8U, 0xFA5656ACU, 0x07F4F4F3U, 0x25EAEACFU, + 0xAF6565CAU, 0x8E7A7AF4U, 0xE9AEAE47U, 0x18080810U, + 0xD5BABA6FU, 0x887878F0U, 0x6F25254AU, 0x722E2E5CU, + 0x241C1C38U, 0xF1A6A657U, 0xC7B4B473U, 0x51C6C697U, + 0x23E8E8CBU, 0x7CDDDDA1U, 0x9C7474E8U, 0x211F1F3EU, + 0xDD4B4B96U, 0xDCBDBD61U, 0x868B8B0DU, 0x858A8A0FU, + 0x907070E0U, 0x423E3E7CU, 0xC4B5B571U, 0xAA6666CCU, + 0xD8484890U, 0x05030306U, 0x01F6F6F7U, 0x120E0E1CU, + 0xA36161C2U, 0x5F35356AU, 0xF95757AEU, 0xD0B9B969U, + 0x91868617U, 0x58C1C199U, 0x271D1D3AU, 0xB99E9E27U, + 0x38E1E1D9U, 0x13F8F8EBU, 0xB398982BU, 0x33111122U, + 0xBB6969D2U, 0x70D9D9A9U, 0x898E8E07U, 0xA7949433U, + 0xB69B9B2DU, 0x221E1E3CU, 0x92878715U, 0x20E9E9C9U, + 0x49CECE87U, 0xFF5555AAU, 0x78282850U, 0x7ADFDFA5U, + 0x8F8C8C03U, 0xF8A1A159U, 0x80898909U, 0x170D0D1AU, + 0xDABFBF65U, 0x31E6E6D7U, 0xC6424284U, 0xB86868D0U, + 0xC3414182U, 0xB0999929U, 0x772D2D5AU, 0x110F0F1EU, + 0xCBB0B07BU, 0xFC5454A8U, 0xD6BBBB6DU, 0x3A16162CU +}; + +#define BYTE(x, y) (xmrig_amd_bfe((x), (y) << 3U, 8U)) + +inline uint4 AES_Round_bittube2(const __local uint *AES0, const __local uint *AES1, uint4 x, uint4 k) +{ + x = ~x; + k.s0 ^= AES0[BYTE(x.s0, 0)] ^ AES1[BYTE(x.s1, 1)] ^ rotate(AES0[BYTE(x.s2, 2)] ^ AES1[BYTE(x.s3, 3)], 16U); + x.s0 ^= k.s0; + k.s1 ^= AES0[BYTE(x.s1, 0)] ^ AES1[BYTE(x.s2, 1)] ^ rotate(AES0[BYTE(x.s3, 2)] ^ AES1[BYTE(x.s0, 3)], 16U); + x.s1 ^= k.s1; + k.s2 ^= AES0[BYTE(x.s2, 0)] ^ AES1[BYTE(x.s3, 1)] ^ rotate(AES0[BYTE(x.s0, 2)] ^ AES1[BYTE(x.s1, 3)], 16U); + x.s2 ^= k.s2; + k.s3 ^= AES0[BYTE(x.s3, 0)] ^ AES1[BYTE(x.s0, 1)] ^ rotate(AES0[BYTE(x.s1, 2)] ^ AES1[BYTE(x.s2, 3)], 16U); + return k; +} + +uint4 AES_Round(const __local uint *AES0, const __local uint *AES1, const __local uint *AES2, const __local uint *AES3, const uint4 X, uint4 key) +{ + key.s0 ^= AES0[BYTE(X.s0, 0)] ^ AES1[BYTE(X.s1, 1)] ^ AES2[BYTE(X.s2, 2)] ^ AES3[BYTE(X.s3, 3)]; + key.s1 ^= AES0[BYTE(X.s1, 0)] ^ AES1[BYTE(X.s2, 1)] ^ AES2[BYTE(X.s3, 2)] ^ AES3[BYTE(X.s0, 3)]; + key.s2 ^= AES0[BYTE(X.s2, 0)] ^ AES1[BYTE(X.s3, 1)] ^ AES2[BYTE(X.s0, 2)] ^ AES3[BYTE(X.s1, 3)]; + key.s3 ^= AES0[BYTE(X.s3, 0)] ^ AES1[BYTE(X.s0, 1)] ^ AES2[BYTE(X.s1, 2)] ^ AES3[BYTE(X.s2, 3)]; + + return key; +} + +uint4 AES_Round_Two_Tables(const __local uint *AES0, const __local uint *AES1, const uint4 X, uint4 key) +{ + key.s0 ^= AES0[BYTE(X.s0, 0)] ^ AES1[BYTE(X.s1, 1)] ^ rotate(AES0[BYTE(X.s2, 2)] ^ AES1[BYTE(X.s3, 3)], 16U); + key.s1 ^= AES0[BYTE(X.s1, 0)] ^ AES1[BYTE(X.s2, 1)] ^ rotate(AES0[BYTE(X.s3, 2)] ^ AES1[BYTE(X.s0, 3)], 16U); + key.s2 ^= AES0[BYTE(X.s2, 0)] ^ AES1[BYTE(X.s3, 1)] ^ rotate(AES0[BYTE(X.s0, 2)] ^ AES1[BYTE(X.s1, 3)], 16U); + key.s3 ^= AES0[BYTE(X.s3, 0)] ^ AES1[BYTE(X.s0, 1)] ^ rotate(AES0[BYTE(X.s1, 2)] ^ AES1[BYTE(X.s2, 3)], 16U); + + return key; +} + +#endif + +)===" diff --git a/src/backend/opencl/cl/cn/wolf-skein.cl b/src/backend/opencl/cl/cn/wolf-skein.cl new file mode 100644 index 00000000..35af3dad --- /dev/null +++ b/src/backend/opencl/cl/cn/wolf-skein.cl @@ -0,0 +1,141 @@ +R"===( +#ifndef WOLF_SKEIN_CL +#define WOLF_SKEIN_CL + +#ifdef cl_amd_media_ops +# pragma OPENCL EXTENSION cl_amd_media_ops : enable +# define xmrig_amd_bitalign(src0, src1, src2) amd_bitalign(src0, src1, src2) +#else +/* taken from https://www.khronos.org/registry/OpenCL/extensions/amd/cl_amd_media_ops.txt + * Build-in Function + * uintn amd_bitalign (uintn src0, uintn src1, uintn src2) + * Description + * dst.s0 = (uint) (((((long)src0.s0) << 32) | (long)src1.s0) >> (src2.s0 & 31)) + * similar operation applied to other components of the vectors. + * + * The implemented function is modified because the last is in our case always a scalar. + * We can ignore the bitwise AND operation. + */ +inline uint2 xmrig_amd_bitalign(const uint2 src0, const uint2 src1, const uint src2) +{ + uint2 result; + result.s0 = (uint) (((((long)src0.s0) << 32) | (long)src1.s0) >> (src2)); + result.s1 = (uint) (((((long)src0.s1) << 32) | (long)src1.s1) >> (src2)); + return result; +} +#endif + +// Vectorized Skein implementation macros and functions by Wolf + +#define SKEIN_KS_PARITY 0x1BD11BDAA9FC1A22 + +static const __constant ulong SKEIN256_IV[8] = +{ + 0xCCD044A12FDB3E13UL, 0xE83590301A79A9EBUL, + 0x55AEA0614F816E6FUL, 0x2A2767A4AE9B94DBUL, + 0xEC06025E74DD7683UL, 0xE7A436CDC4746251UL, + 0xC36FBAF9393AD185UL, 0x3EEDBA1833EDFC13UL +}; + +static const __constant ulong SKEIN512_256_IV[8] = +{ + 0xCCD044A12FDB3E13UL, 0xE83590301A79A9EBUL, + 0x55AEA0614F816E6FUL, 0x2A2767A4AE9B94DBUL, + 0xEC06025E74DD7683UL, 0xE7A436CDC4746251UL, + 0xC36FBAF9393AD185UL, 0x3EEDBA1833EDFC13UL +}; + +#define SKEIN_INJECT_KEY(p, s) do { \ + p += h; \ + p.s5 += t[s % 3]; \ + p.s6 += t[(s + 1) % 3]; \ + p.s7 += s; \ +} while(0) + +ulong SKEIN_ROT(const uint2 x, const uint y) +{ + if (y < 32) { + return(as_ulong(xmrig_amd_bitalign(x, x.s10, 32 - y))); + } + else { + return(as_ulong(xmrig_amd_bitalign(x.s10, x, 32 - (y - 32)))); + } +} + +void SkeinMix8(ulong4 *pv0, ulong4 *pv1, const uint rc0, const uint rc1, const uint rc2, const uint rc3) +{ + *pv0 += *pv1; + (*pv1).s0 = SKEIN_ROT(as_uint2((*pv1).s0), rc0); + (*pv1).s1 = SKEIN_ROT(as_uint2((*pv1).s1), rc1); + (*pv1).s2 = SKEIN_ROT(as_uint2((*pv1).s2), rc2); + (*pv1).s3 = SKEIN_ROT(as_uint2((*pv1).s3), rc3); + *pv1 ^= *pv0; +} + +ulong8 SkeinEvenRound(ulong8 p, const ulong8 h, const ulong *t, const uint s) +{ + SKEIN_INJECT_KEY(p, s); + ulong4 pv0 = p.even, pv1 = p.odd; + + SkeinMix8(&pv0, &pv1, 46, 36, 19, 37); + pv0 = shuffle(pv0, (ulong4)(1, 2, 3, 0)); + pv1 = shuffle(pv1, (ulong4)(0, 3, 2, 1)); + + SkeinMix8(&pv0, &pv1, 33, 27, 14, 42); + pv0 = shuffle(pv0, (ulong4)(1, 2, 3, 0)); + pv1 = shuffle(pv1, (ulong4)(0, 3, 2, 1)); + + SkeinMix8(&pv0, &pv1, 17, 49, 36, 39); + pv0 = shuffle(pv0, (ulong4)(1, 2, 3, 0)); + pv1 = shuffle(pv1, (ulong4)(0, 3, 2, 1)); + + SkeinMix8(&pv0, &pv1, 44, 9, 54, 56); + return(shuffle2(pv0, pv1, (ulong8)(1, 4, 2, 7, 3, 6, 0, 5))); +} + +ulong8 SkeinOddRound(ulong8 p, const ulong8 h, const ulong *t, const uint s) +{ + SKEIN_INJECT_KEY(p, s); + ulong4 pv0 = p.even, pv1 = p.odd; + + SkeinMix8(&pv0, &pv1, 39, 30, 34, 24); + pv0 = shuffle(pv0, (ulong4)(1, 2, 3, 0)); + pv1 = shuffle(pv1, (ulong4)(0, 3, 2, 1)); + + SkeinMix8(&pv0, &pv1, 13, 50, 10, 17); + pv0 = shuffle(pv0, (ulong4)(1, 2, 3, 0)); + pv1 = shuffle(pv1, (ulong4)(0, 3, 2, 1)); + + SkeinMix8(&pv0, &pv1, 25, 29, 39, 43); + pv0 = shuffle(pv0, (ulong4)(1, 2, 3, 0)); + pv1 = shuffle(pv1, (ulong4)(0, 3, 2, 1)); + + SkeinMix8(&pv0, &pv1, 8, 35, 56, 22); + return(shuffle2(pv0, pv1, (ulong8)(1, 4, 2, 7, 3, 6, 0, 5))); +} + +ulong8 Skein512Block(ulong8 p, ulong8 h, ulong h8, const ulong *t) +{ + #pragma unroll + for(int i = 0; i < 18; ++i) + { + p = SkeinEvenRound(p, h, t, i); + ++i; + ulong tmp = h.s0; + h = shuffle(h, (ulong8)(1, 2, 3, 4, 5, 6, 7, 0)); + h.s7 = h8; + h8 = tmp; + p = SkeinOddRound(p, h, t, i); + tmp = h.s0; + h = shuffle(h, (ulong8)(1, 2, 3, 4, 5, 6, 7, 0)); + h.s7 = h8; + h8 = tmp; + } + + SKEIN_INJECT_KEY(p, 18); + return(p); +} + +#endif + +)===" diff --git a/src/backend/opencl/cl/rx/aes.cl b/src/backend/opencl/cl/rx/aes.cl new file mode 100644 index 00000000..bb49f552 --- /dev/null +++ b/src/backend/opencl/cl/rx/aes.cl @@ -0,0 +1,642 @@ +R"===( +/* +Copyright (c) 2019 SChernykh + +This file is part of RandomX OpenCL. + +RandomX OpenCL is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +RandomX OpenCL is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with RandomX OpenCL. If not, see . +*/ + +__constant static const uint AES_TABLE[2048] = +{ + 0xa56363c6U, 0x847c7cf8U, 0x997777eeU, 0x8d7b7bf6U, + 0x0df2f2ffU, 0xbd6b6bd6U, 0xb16f6fdeU, 0x54c5c591U, + 0x50303060U, 0x03010102U, 0xa96767ceU, 0x7d2b2b56U, + 0x19fefee7U, 0x62d7d7b5U, 0xe6abab4dU, 0x9a7676ecU, + 0x45caca8fU, 0x9d82821fU, 0x40c9c989U, 0x877d7dfaU, + 0x15fafaefU, 0xeb5959b2U, 0xc947478eU, 0x0bf0f0fbU, + 0xecadad41U, 0x67d4d4b3U, 0xfda2a25fU, 0xeaafaf45U, + 0xbf9c9c23U, 0xf7a4a453U, 0x967272e4U, 0x5bc0c09bU, + 0xc2b7b775U, 0x1cfdfde1U, 0xae93933dU, 0x6a26264cU, + 0x5a36366cU, 0x413f3f7eU, 0x02f7f7f5U, 0x4fcccc83U, + 0x5c343468U, 0xf4a5a551U, 0x34e5e5d1U, 0x08f1f1f9U, + 0x937171e2U, 0x73d8d8abU, 0x53313162U, 0x3f15152aU, + 0x0c040408U, 0x52c7c795U, 0x65232346U, 0x5ec3c39dU, + 0x28181830U, 0xa1969637U, 0x0f05050aU, 0xb59a9a2fU, + 0x0907070eU, 0x36121224U, 0x9b80801bU, 0x3de2e2dfU, + 0x26ebebcdU, 0x6927274eU, 0xcdb2b27fU, 0x9f7575eaU, + 0x1b090912U, 0x9e83831dU, 0x742c2c58U, 0x2e1a1a34U, + 0x2d1b1b36U, 0xb26e6edcU, 0xee5a5ab4U, 0xfba0a05bU, + 0xf65252a4U, 0x4d3b3b76U, 0x61d6d6b7U, 0xceb3b37dU, + 0x7b292952U, 0x3ee3e3ddU, 0x712f2f5eU, 0x97848413U, + 0xf55353a6U, 0x68d1d1b9U, 0x00000000U, 0x2cededc1U, + 0x60202040U, 0x1ffcfce3U, 0xc8b1b179U, 0xed5b5bb6U, + 0xbe6a6ad4U, 0x46cbcb8dU, 0xd9bebe67U, 0x4b393972U, + 0xde4a4a94U, 0xd44c4c98U, 0xe85858b0U, 0x4acfcf85U, + 0x6bd0d0bbU, 0x2aefefc5U, 0xe5aaaa4fU, 0x16fbfbedU, + 0xc5434386U, 0xd74d4d9aU, 0x55333366U, 0x94858511U, + 0xcf45458aU, 0x10f9f9e9U, 0x06020204U, 0x817f7ffeU, + 0xf05050a0U, 0x443c3c78U, 0xba9f9f25U, 0xe3a8a84bU, + 0xf35151a2U, 0xfea3a35dU, 0xc0404080U, 0x8a8f8f05U, + 0xad92923fU, 0xbc9d9d21U, 0x48383870U, 0x04f5f5f1U, + 0xdfbcbc63U, 0xc1b6b677U, 0x75dadaafU, 0x63212142U, + 0x30101020U, 0x1affffe5U, 0x0ef3f3fdU, 0x6dd2d2bfU, + 0x4ccdcd81U, 0x140c0c18U, 0x35131326U, 0x2fececc3U, + 0xe15f5fbeU, 0xa2979735U, 0xcc444488U, 0x3917172eU, + 0x57c4c493U, 0xf2a7a755U, 0x827e7efcU, 0x473d3d7aU, + 0xac6464c8U, 0xe75d5dbaU, 0x2b191932U, 0x957373e6U, + 0xa06060c0U, 0x98818119U, 0xd14f4f9eU, 0x7fdcdca3U, + 0x66222244U, 0x7e2a2a54U, 0xab90903bU, 0x8388880bU, + 0xca46468cU, 0x29eeeec7U, 0xd3b8b86bU, 0x3c141428U, + 0x79dedea7U, 0xe25e5ebcU, 0x1d0b0b16U, 0x76dbdbadU, + 0x3be0e0dbU, 0x56323264U, 0x4e3a3a74U, 0x1e0a0a14U, + 0xdb494992U, 0x0a06060cU, 0x6c242448U, 0xe45c5cb8U, + 0x5dc2c29fU, 0x6ed3d3bdU, 0xefacac43U, 0xa66262c4U, + 0xa8919139U, 0xa4959531U, 0x37e4e4d3U, 0x8b7979f2U, + 0x32e7e7d5U, 0x43c8c88bU, 0x5937376eU, 0xb76d6ddaU, + 0x8c8d8d01U, 0x64d5d5b1U, 0xd24e4e9cU, 0xe0a9a949U, + 0xb46c6cd8U, 0xfa5656acU, 0x07f4f4f3U, 0x25eaeacfU, + 0xaf6565caU, 0x8e7a7af4U, 0xe9aeae47U, 0x18080810U, + 0xd5baba6fU, 0x887878f0U, 0x6f25254aU, 0x722e2e5cU, + 0x241c1c38U, 0xf1a6a657U, 0xc7b4b473U, 0x51c6c697U, + 0x23e8e8cbU, 0x7cdddda1U, 0x9c7474e8U, 0x211f1f3eU, + 0xdd4b4b96U, 0xdcbdbd61U, 0x868b8b0dU, 0x858a8a0fU, + 0x907070e0U, 0x423e3e7cU, 0xc4b5b571U, 0xaa6666ccU, + 0xd8484890U, 0x05030306U, 0x01f6f6f7U, 0x120e0e1cU, + 0xa36161c2U, 0x5f35356aU, 0xf95757aeU, 0xd0b9b969U, + 0x91868617U, 0x58c1c199U, 0x271d1d3aU, 0xb99e9e27U, + 0x38e1e1d9U, 0x13f8f8ebU, 0xb398982bU, 0x33111122U, + 0xbb6969d2U, 0x70d9d9a9U, 0x898e8e07U, 0xa7949433U, + 0xb69b9b2dU, 0x221e1e3cU, 0x92878715U, 0x20e9e9c9U, + 0x49cece87U, 0xff5555aaU, 0x78282850U, 0x7adfdfa5U, + 0x8f8c8c03U, 0xf8a1a159U, 0x80898909U, 0x170d0d1aU, + 0xdabfbf65U, 0x31e6e6d7U, 0xc6424284U, 0xb86868d0U, + 0xc3414182U, 0xb0999929U, 0x772d2d5aU, 0x110f0f1eU, + 0xcbb0b07bU, 0xfc5454a8U, 0xd6bbbb6dU, 0x3a16162cU, + 0x6363c6a5U, 0x7c7cf884U, 0x7777ee99U, 0x7b7bf68dU, + 0xf2f2ff0dU, 0x6b6bd6bdU, 0x6f6fdeb1U, 0xc5c59154U, + 0x30306050U, 0x01010203U, 0x6767cea9U, 0x2b2b567dU, + 0xfefee719U, 0xd7d7b562U, 0xabab4de6U, 0x7676ec9aU, + 0xcaca8f45U, 0x82821f9dU, 0xc9c98940U, 0x7d7dfa87U, + 0xfafaef15U, 0x5959b2ebU, 0x47478ec9U, 0xf0f0fb0bU, + 0xadad41ecU, 0xd4d4b367U, 0xa2a25ffdU, 0xafaf45eaU, + 0x9c9c23bfU, 0xa4a453f7U, 0x7272e496U, 0xc0c09b5bU, + 0xb7b775c2U, 0xfdfde11cU, 0x93933daeU, 0x26264c6aU, + 0x36366c5aU, 0x3f3f7e41U, 0xf7f7f502U, 0xcccc834fU, + 0x3434685cU, 0xa5a551f4U, 0xe5e5d134U, 0xf1f1f908U, + 0x7171e293U, 0xd8d8ab73U, 0x31316253U, 0x15152a3fU, + 0x0404080cU, 0xc7c79552U, 0x23234665U, 0xc3c39d5eU, + 0x18183028U, 0x969637a1U, 0x05050a0fU, 0x9a9a2fb5U, + 0x07070e09U, 0x12122436U, 0x80801b9bU, 0xe2e2df3dU, + 0xebebcd26U, 0x27274e69U, 0xb2b27fcdU, 0x7575ea9fU, + 0x0909121bU, 0x83831d9eU, 0x2c2c5874U, 0x1a1a342eU, + 0x1b1b362dU, 0x6e6edcb2U, 0x5a5ab4eeU, 0xa0a05bfbU, + 0x5252a4f6U, 0x3b3b764dU, 0xd6d6b761U, 0xb3b37dceU, + 0x2929527bU, 0xe3e3dd3eU, 0x2f2f5e71U, 0x84841397U, + 0x5353a6f5U, 0xd1d1b968U, 0x00000000U, 0xededc12cU, + 0x20204060U, 0xfcfce31fU, 0xb1b179c8U, 0x5b5bb6edU, + 0x6a6ad4beU, 0xcbcb8d46U, 0xbebe67d9U, 0x3939724bU, + 0x4a4a94deU, 0x4c4c98d4U, 0x5858b0e8U, 0xcfcf854aU, + 0xd0d0bb6bU, 0xefefc52aU, 0xaaaa4fe5U, 0xfbfbed16U, + 0x434386c5U, 0x4d4d9ad7U, 0x33336655U, 0x85851194U, + 0x45458acfU, 0xf9f9e910U, 0x02020406U, 0x7f7ffe81U, + 0x5050a0f0U, 0x3c3c7844U, 0x9f9f25baU, 0xa8a84be3U, + 0x5151a2f3U, 0xa3a35dfeU, 0x404080c0U, 0x8f8f058aU, + 0x92923fadU, 0x9d9d21bcU, 0x38387048U, 0xf5f5f104U, + 0xbcbc63dfU, 0xb6b677c1U, 0xdadaaf75U, 0x21214263U, + 0x10102030U, 0xffffe51aU, 0xf3f3fd0eU, 0xd2d2bf6dU, + 0xcdcd814cU, 0x0c0c1814U, 0x13132635U, 0xececc32fU, + 0x5f5fbee1U, 0x979735a2U, 0x444488ccU, 0x17172e39U, + 0xc4c49357U, 0xa7a755f2U, 0x7e7efc82U, 0x3d3d7a47U, + 0x6464c8acU, 0x5d5dbae7U, 0x1919322bU, 0x7373e695U, + 0x6060c0a0U, 0x81811998U, 0x4f4f9ed1U, 0xdcdca37fU, + 0x22224466U, 0x2a2a547eU, 0x90903babU, 0x88880b83U, + 0x46468ccaU, 0xeeeec729U, 0xb8b86bd3U, 0x1414283cU, + 0xdedea779U, 0x5e5ebce2U, 0x0b0b161dU, 0xdbdbad76U, + 0xe0e0db3bU, 0x32326456U, 0x3a3a744eU, 0x0a0a141eU, + 0x494992dbU, 0x06060c0aU, 0x2424486cU, 0x5c5cb8e4U, + 0xc2c29f5dU, 0xd3d3bd6eU, 0xacac43efU, 0x6262c4a6U, + 0x919139a8U, 0x959531a4U, 0xe4e4d337U, 0x7979f28bU, + 0xe7e7d532U, 0xc8c88b43U, 0x37376e59U, 0x6d6ddab7U, + 0x8d8d018cU, 0xd5d5b164U, 0x4e4e9cd2U, 0xa9a949e0U, + 0x6c6cd8b4U, 0x5656acfaU, 0xf4f4f307U, 0xeaeacf25U, + 0x6565caafU, 0x7a7af48eU, 0xaeae47e9U, 0x08081018U, + 0xbaba6fd5U, 0x7878f088U, 0x25254a6fU, 0x2e2e5c72U, + 0x1c1c3824U, 0xa6a657f1U, 0xb4b473c7U, 0xc6c69751U, + 0xe8e8cb23U, 0xdddda17cU, 0x7474e89cU, 0x1f1f3e21U, + 0x4b4b96ddU, 0xbdbd61dcU, 0x8b8b0d86U, 0x8a8a0f85U, + 0x7070e090U, 0x3e3e7c42U, 0xb5b571c4U, 0x6666ccaaU, + 0x484890d8U, 0x03030605U, 0xf6f6f701U, 0x0e0e1c12U, + 0x6161c2a3U, 0x35356a5fU, 0x5757aef9U, 0xb9b969d0U, + 0x86861791U, 0xc1c19958U, 0x1d1d3a27U, 0x9e9e27b9U, + 0xe1e1d938U, 0xf8f8eb13U, 0x98982bb3U, 0x11112233U, + 0x6969d2bbU, 0xd9d9a970U, 0x8e8e0789U, 0x949433a7U, + 0x9b9b2db6U, 0x1e1e3c22U, 0x87871592U, 0xe9e9c920U, + 0xcece8749U, 0x5555aaffU, 0x28285078U, 0xdfdfa57aU, + 0x8c8c038fU, 0xa1a159f8U, 0x89890980U, 0x0d0d1a17U, + 0xbfbf65daU, 0xe6e6d731U, 0x424284c6U, 0x6868d0b8U, + 0x414182c3U, 0x999929b0U, 0x2d2d5a77U, 0x0f0f1e11U, + 0xb0b07bcbU, 0x5454a8fcU, 0xbbbb6dd6U, 0x16162c3aU, + 0x63c6a563U, 0x7cf8847cU, 0x77ee9977U, 0x7bf68d7bU, + 0xf2ff0df2U, 0x6bd6bd6bU, 0x6fdeb16fU, 0xc59154c5U, + 0x30605030U, 0x01020301U, 0x67cea967U, 0x2b567d2bU, + 0xfee719feU, 0xd7b562d7U, 0xab4de6abU, 0x76ec9a76U, + 0xca8f45caU, 0x821f9d82U, 0xc98940c9U, 0x7dfa877dU, + 0xfaef15faU, 0x59b2eb59U, 0x478ec947U, 0xf0fb0bf0U, + 0xad41ecadU, 0xd4b367d4U, 0xa25ffda2U, 0xaf45eaafU, + 0x9c23bf9cU, 0xa453f7a4U, 0x72e49672U, 0xc09b5bc0U, + 0xb775c2b7U, 0xfde11cfdU, 0x933dae93U, 0x264c6a26U, + 0x366c5a36U, 0x3f7e413fU, 0xf7f502f7U, 0xcc834fccU, + 0x34685c34U, 0xa551f4a5U, 0xe5d134e5U, 0xf1f908f1U, + 0x71e29371U, 0xd8ab73d8U, 0x31625331U, 0x152a3f15U, + 0x04080c04U, 0xc79552c7U, 0x23466523U, 0xc39d5ec3U, + 0x18302818U, 0x9637a196U, 0x050a0f05U, 0x9a2fb59aU, + 0x070e0907U, 0x12243612U, 0x801b9b80U, 0xe2df3de2U, + 0xebcd26ebU, 0x274e6927U, 0xb27fcdb2U, 0x75ea9f75U, + 0x09121b09U, 0x831d9e83U, 0x2c58742cU, 0x1a342e1aU, + 0x1b362d1bU, 0x6edcb26eU, 0x5ab4ee5aU, 0xa05bfba0U, + 0x52a4f652U, 0x3b764d3bU, 0xd6b761d6U, 0xb37dceb3U, + 0x29527b29U, 0xe3dd3ee3U, 0x2f5e712fU, 0x84139784U, + 0x53a6f553U, 0xd1b968d1U, 0x00000000U, 0xedc12cedU, + 0x20406020U, 0xfce31ffcU, 0xb179c8b1U, 0x5bb6ed5bU, + 0x6ad4be6aU, 0xcb8d46cbU, 0xbe67d9beU, 0x39724b39U, + 0x4a94de4aU, 0x4c98d44cU, 0x58b0e858U, 0xcf854acfU, + 0xd0bb6bd0U, 0xefc52aefU, 0xaa4fe5aaU, 0xfbed16fbU, + 0x4386c543U, 0x4d9ad74dU, 0x33665533U, 0x85119485U, + 0x458acf45U, 0xf9e910f9U, 0x02040602U, 0x7ffe817fU, + 0x50a0f050U, 0x3c78443cU, 0x9f25ba9fU, 0xa84be3a8U, + 0x51a2f351U, 0xa35dfea3U, 0x4080c040U, 0x8f058a8fU, + 0x923fad92U, 0x9d21bc9dU, 0x38704838U, 0xf5f104f5U, + 0xbc63dfbcU, 0xb677c1b6U, 0xdaaf75daU, 0x21426321U, + 0x10203010U, 0xffe51affU, 0xf3fd0ef3U, 0xd2bf6dd2U, + 0xcd814ccdU, 0x0c18140cU, 0x13263513U, 0xecc32fecU, + 0x5fbee15fU, 0x9735a297U, 0x4488cc44U, 0x172e3917U, + 0xc49357c4U, 0xa755f2a7U, 0x7efc827eU, 0x3d7a473dU, + 0x64c8ac64U, 0x5dbae75dU, 0x19322b19U, 0x73e69573U, + 0x60c0a060U, 0x81199881U, 0x4f9ed14fU, 0xdca37fdcU, + 0x22446622U, 0x2a547e2aU, 0x903bab90U, 0x880b8388U, + 0x468cca46U, 0xeec729eeU, 0xb86bd3b8U, 0x14283c14U, + 0xdea779deU, 0x5ebce25eU, 0x0b161d0bU, 0xdbad76dbU, + 0xe0db3be0U, 0x32645632U, 0x3a744e3aU, 0x0a141e0aU, + 0x4992db49U, 0x060c0a06U, 0x24486c24U, 0x5cb8e45cU, + 0xc29f5dc2U, 0xd3bd6ed3U, 0xac43efacU, 0x62c4a662U, + 0x9139a891U, 0x9531a495U, 0xe4d337e4U, 0x79f28b79U, + 0xe7d532e7U, 0xc88b43c8U, 0x376e5937U, 0x6ddab76dU, + 0x8d018c8dU, 0xd5b164d5U, 0x4e9cd24eU, 0xa949e0a9U, + 0x6cd8b46cU, 0x56acfa56U, 0xf4f307f4U, 0xeacf25eaU, + 0x65caaf65U, 0x7af48e7aU, 0xae47e9aeU, 0x08101808U, + 0xba6fd5baU, 0x78f08878U, 0x254a6f25U, 0x2e5c722eU, + 0x1c38241cU, 0xa657f1a6U, 0xb473c7b4U, 0xc69751c6U, + 0xe8cb23e8U, 0xdda17cddU, 0x74e89c74U, 0x1f3e211fU, + 0x4b96dd4bU, 0xbd61dcbdU, 0x8b0d868bU, 0x8a0f858aU, + 0x70e09070U, 0x3e7c423eU, 0xb571c4b5U, 0x66ccaa66U, + 0x4890d848U, 0x03060503U, 0xf6f701f6U, 0x0e1c120eU, + 0x61c2a361U, 0x356a5f35U, 0x57aef957U, 0xb969d0b9U, + 0x86179186U, 0xc19958c1U, 0x1d3a271dU, 0x9e27b99eU, + 0xe1d938e1U, 0xf8eb13f8U, 0x982bb398U, 0x11223311U, + 0x69d2bb69U, 0xd9a970d9U, 0x8e07898eU, 0x9433a794U, + 0x9b2db69bU, 0x1e3c221eU, 0x87159287U, 0xe9c920e9U, + 0xce8749ceU, 0x55aaff55U, 0x28507828U, 0xdfa57adfU, + 0x8c038f8cU, 0xa159f8a1U, 0x89098089U, 0x0d1a170dU, + 0xbf65dabfU, 0xe6d731e6U, 0x4284c642U, 0x68d0b868U, + 0x4182c341U, 0x9929b099U, 0x2d5a772dU, 0x0f1e110fU, + 0xb07bcbb0U, 0x54a8fc54U, 0xbb6dd6bbU, 0x162c3a16U, + 0xc6a56363U, 0xf8847c7cU, 0xee997777U, 0xf68d7b7bU, + 0xff0df2f2U, 0xd6bd6b6bU, 0xdeb16f6fU, 0x9154c5c5U, + 0x60503030U, 0x02030101U, 0xcea96767U, 0x567d2b2bU, + 0xe719fefeU, 0xb562d7d7U, 0x4de6ababU, 0xec9a7676U, + 0x8f45cacaU, 0x1f9d8282U, 0x8940c9c9U, 0xfa877d7dU, + 0xef15fafaU, 0xb2eb5959U, 0x8ec94747U, 0xfb0bf0f0U, + 0x41ecadadU, 0xb367d4d4U, 0x5ffda2a2U, 0x45eaafafU, + 0x23bf9c9cU, 0x53f7a4a4U, 0xe4967272U, 0x9b5bc0c0U, + 0x75c2b7b7U, 0xe11cfdfdU, 0x3dae9393U, 0x4c6a2626U, + 0x6c5a3636U, 0x7e413f3fU, 0xf502f7f7U, 0x834fccccU, + 0x685c3434U, 0x51f4a5a5U, 0xd134e5e5U, 0xf908f1f1U, + 0xe2937171U, 0xab73d8d8U, 0x62533131U, 0x2a3f1515U, + 0x080c0404U, 0x9552c7c7U, 0x46652323U, 0x9d5ec3c3U, + 0x30281818U, 0x37a19696U, 0x0a0f0505U, 0x2fb59a9aU, + 0x0e090707U, 0x24361212U, 0x1b9b8080U, 0xdf3de2e2U, + 0xcd26ebebU, 0x4e692727U, 0x7fcdb2b2U, 0xea9f7575U, + 0x121b0909U, 0x1d9e8383U, 0x58742c2cU, 0x342e1a1aU, + 0x362d1b1bU, 0xdcb26e6eU, 0xb4ee5a5aU, 0x5bfba0a0U, + 0xa4f65252U, 0x764d3b3bU, 0xb761d6d6U, 0x7dceb3b3U, + 0x527b2929U, 0xdd3ee3e3U, 0x5e712f2fU, 0x13978484U, + 0xa6f55353U, 0xb968d1d1U, 0x00000000U, 0xc12cededU, + 0x40602020U, 0xe31ffcfcU, 0x79c8b1b1U, 0xb6ed5b5bU, + 0xd4be6a6aU, 0x8d46cbcbU, 0x67d9bebeU, 0x724b3939U, + 0x94de4a4aU, 0x98d44c4cU, 0xb0e85858U, 0x854acfcfU, + 0xbb6bd0d0U, 0xc52aefefU, 0x4fe5aaaaU, 0xed16fbfbU, + 0x86c54343U, 0x9ad74d4dU, 0x66553333U, 0x11948585U, + 0x8acf4545U, 0xe910f9f9U, 0x04060202U, 0xfe817f7fU, + 0xa0f05050U, 0x78443c3cU, 0x25ba9f9fU, 0x4be3a8a8U, + 0xa2f35151U, 0x5dfea3a3U, 0x80c04040U, 0x058a8f8fU, + 0x3fad9292U, 0x21bc9d9dU, 0x70483838U, 0xf104f5f5U, + 0x63dfbcbcU, 0x77c1b6b6U, 0xaf75dadaU, 0x42632121U, + 0x20301010U, 0xe51affffU, 0xfd0ef3f3U, 0xbf6dd2d2U, + 0x814ccdcdU, 0x18140c0cU, 0x26351313U, 0xc32fececU, + 0xbee15f5fU, 0x35a29797U, 0x88cc4444U, 0x2e391717U, + 0x9357c4c4U, 0x55f2a7a7U, 0xfc827e7eU, 0x7a473d3dU, + 0xc8ac6464U, 0xbae75d5dU, 0x322b1919U, 0xe6957373U, + 0xc0a06060U, 0x19988181U, 0x9ed14f4fU, 0xa37fdcdcU, + 0x44662222U, 0x547e2a2aU, 0x3bab9090U, 0x0b838888U, + 0x8cca4646U, 0xc729eeeeU, 0x6bd3b8b8U, 0x283c1414U, + 0xa779dedeU, 0xbce25e5eU, 0x161d0b0bU, 0xad76dbdbU, + 0xdb3be0e0U, 0x64563232U, 0x744e3a3aU, 0x141e0a0aU, + 0x92db4949U, 0x0c0a0606U, 0x486c2424U, 0xb8e45c5cU, + 0x9f5dc2c2U, 0xbd6ed3d3U, 0x43efacacU, 0xc4a66262U, + 0x39a89191U, 0x31a49595U, 0xd337e4e4U, 0xf28b7979U, + 0xd532e7e7U, 0x8b43c8c8U, 0x6e593737U, 0xdab76d6dU, + 0x018c8d8dU, 0xb164d5d5U, 0x9cd24e4eU, 0x49e0a9a9U, + 0xd8b46c6cU, 0xacfa5656U, 0xf307f4f4U, 0xcf25eaeaU, + 0xcaaf6565U, 0xf48e7a7aU, 0x47e9aeaeU, 0x10180808U, + 0x6fd5babaU, 0xf0887878U, 0x4a6f2525U, 0x5c722e2eU, + 0x38241c1cU, 0x57f1a6a6U, 0x73c7b4b4U, 0x9751c6c6U, + 0xcb23e8e8U, 0xa17cddddU, 0xe89c7474U, 0x3e211f1fU, + 0x96dd4b4bU, 0x61dcbdbdU, 0x0d868b8bU, 0x0f858a8aU, + 0xe0907070U, 0x7c423e3eU, 0x71c4b5b5U, 0xccaa6666U, + 0x90d84848U, 0x06050303U, 0xf701f6f6U, 0x1c120e0eU, + 0xc2a36161U, 0x6a5f3535U, 0xaef95757U, 0x69d0b9b9U, + 0x17918686U, 0x9958c1c1U, 0x3a271d1dU, 0x27b99e9eU, + 0xd938e1e1U, 0xeb13f8f8U, 0x2bb39898U, 0x22331111U, + 0xd2bb6969U, 0xa970d9d9U, 0x07898e8eU, 0x33a79494U, + 0x2db69b9bU, 0x3c221e1eU, 0x15928787U, 0xc920e9e9U, + 0x8749ceceU, 0xaaff5555U, 0x50782828U, 0xa57adfdfU, + 0x038f8c8cU, 0x59f8a1a1U, 0x09808989U, 0x1a170d0dU, + 0x65dabfbfU, 0xd731e6e6U, 0x84c64242U, 0xd0b86868U, + 0x82c34141U, 0x29b09999U, 0x5a772d2dU, 0x1e110f0fU, + 0x7bcbb0b0U, 0xa8fc5454U, 0x6dd6bbbbU, 0x2c3a1616U, + 0x50a7f451U, 0x5365417eU, 0xc3a4171aU, 0x965e273aU, + 0xcb6bab3bU, 0xf1459d1fU, 0xab58faacU, 0x9303e34bU, + 0x55fa3020U, 0xf66d76adU, 0x9176cc88U, 0x254c02f5U, + 0xfcd7e54fU, 0xd7cb2ac5U, 0x80443526U, 0x8fa362b5U, + 0x495ab1deU, 0x671bba25U, 0x980eea45U, 0xe1c0fe5dU, + 0x02752fc3U, 0x12f04c81U, 0xa397468dU, 0xc6f9d36bU, + 0xe75f8f03U, 0x959c9215U, 0xeb7a6dbfU, 0xda595295U, + 0x2d83bed4U, 0xd3217458U, 0x2969e049U, 0x44c8c98eU, + 0x6a89c275U, 0x78798ef4U, 0x6b3e5899U, 0xdd71b927U, + 0xb64fe1beU, 0x17ad88f0U, 0x66ac20c9U, 0xb43ace7dU, + 0x184adf63U, 0x82311ae5U, 0x60335197U, 0x457f5362U, + 0xe07764b1U, 0x84ae6bbbU, 0x1ca081feU, 0x942b08f9U, + 0x58684870U, 0x19fd458fU, 0x876cde94U, 0xb7f87b52U, + 0x23d373abU, 0xe2024b72U, 0x578f1fe3U, 0x2aab5566U, + 0x0728ebb2U, 0x03c2b52fU, 0x9a7bc586U, 0xa50837d3U, + 0xf2872830U, 0xb2a5bf23U, 0xba6a0302U, 0x5c8216edU, + 0x2b1ccf8aU, 0x92b479a7U, 0xf0f207f3U, 0xa1e2694eU, + 0xcdf4da65U, 0xd5be0506U, 0x1f6234d1U, 0x8afea6c4U, + 0x9d532e34U, 0xa055f3a2U, 0x32e18a05U, 0x75ebf6a4U, + 0x39ec830bU, 0xaaef6040U, 0x069f715eU, 0x51106ebdU, + 0xf98a213eU, 0x3d06dd96U, 0xae053eddU, 0x46bde64dU, +)===" +R"===( + 0xb58d5491U, 0x055dc471U, 0x6fd40604U, 0xff155060U, + 0x24fb9819U, 0x97e9bdd6U, 0xcc434089U, 0x779ed967U, + 0xbd42e8b0U, 0x888b8907U, 0x385b19e7U, 0xdbeec879U, + 0x470a7ca1U, 0xe90f427cU, 0xc91e84f8U, 0x00000000U, + 0x83868009U, 0x48ed2b32U, 0xac70111eU, 0x4e725a6cU, + 0xfbff0efdU, 0x5638850fU, 0x1ed5ae3dU, 0x27392d36U, + 0x64d90f0aU, 0x21a65c68U, 0xd1545b9bU, 0x3a2e3624U, + 0xb1670a0cU, 0x0fe75793U, 0xd296eeb4U, 0x9e919b1bU, + 0x4fc5c080U, 0xa220dc61U, 0x694b775aU, 0x161a121cU, + 0x0aba93e2U, 0xe52aa0c0U, 0x43e0223cU, 0x1d171b12U, + 0x0b0d090eU, 0xadc78bf2U, 0xb9a8b62dU, 0xc8a91e14U, + 0x8519f157U, 0x4c0775afU, 0xbbdd99eeU, 0xfd607fa3U, + 0x9f2601f7U, 0xbcf5725cU, 0xc53b6644U, 0x347efb5bU, + 0x7629438bU, 0xdcc623cbU, 0x68fcedb6U, 0x63f1e4b8U, + 0xcadc31d7U, 0x10856342U, 0x40229713U, 0x2011c684U, + 0x7d244a85U, 0xf83dbbd2U, 0x1132f9aeU, 0x6da129c7U, + 0x4b2f9e1dU, 0xf330b2dcU, 0xec52860dU, 0xd0e3c177U, + 0x6c16b32bU, 0x99b970a9U, 0xfa489411U, 0x2264e947U, + 0xc48cfca8U, 0x1a3ff0a0U, 0xd82c7d56U, 0xef903322U, + 0xc74e4987U, 0xc1d138d9U, 0xfea2ca8cU, 0x360bd498U, + 0xcf81f5a6U, 0x28de7aa5U, 0x268eb7daU, 0xa4bfad3fU, + 0xe49d3a2cU, 0x0d927850U, 0x9bcc5f6aU, 0x62467e54U, + 0xc2138df6U, 0xe8b8d890U, 0x5ef7392eU, 0xf5afc382U, + 0xbe805d9fU, 0x7c93d069U, 0xa92dd56fU, 0xb31225cfU, + 0x3b99acc8U, 0xa77d1810U, 0x6e639ce8U, 0x7bbb3bdbU, + 0x097826cdU, 0xf418596eU, 0x01b79aecU, 0xa89a4f83U, + 0x656e95e6U, 0x7ee6ffaaU, 0x08cfbc21U, 0xe6e815efU, + 0xd99be7baU, 0xce366f4aU, 0xd4099feaU, 0xd67cb029U, + 0xafb2a431U, 0x31233f2aU, 0x3094a5c6U, 0xc066a235U, + 0x37bc4e74U, 0xa6ca82fcU, 0xb0d090e0U, 0x15d8a733U, + 0x4a9804f1U, 0xf7daec41U, 0x0e50cd7fU, 0x2ff69117U, + 0x8dd64d76U, 0x4db0ef43U, 0x544daaccU, 0xdf0496e4U, + 0xe3b5d19eU, 0x1b886a4cU, 0xb81f2cc1U, 0x7f516546U, + 0x04ea5e9dU, 0x5d358c01U, 0x737487faU, 0x2e410bfbU, + 0x5a1d67b3U, 0x52d2db92U, 0x335610e9U, 0x1347d66dU, + 0x8c61d79aU, 0x7a0ca137U, 0x8e14f859U, 0x893c13ebU, + 0xee27a9ceU, 0x35c961b7U, 0xede51ce1U, 0x3cb1477aU, + 0x59dfd29cU, 0x3f73f255U, 0x79ce1418U, 0xbf37c773U, + 0xeacdf753U, 0x5baafd5fU, 0x146f3ddfU, 0x86db4478U, + 0x81f3afcaU, 0x3ec468b9U, 0x2c342438U, 0x5f40a3c2U, + 0x72c31d16U, 0x0c25e2bcU, 0x8b493c28U, 0x41950dffU, + 0x7101a839U, 0xdeb30c08U, 0x9ce4b4d8U, 0x90c15664U, + 0x6184cb7bU, 0x70b632d5U, 0x745c6c48U, 0x4257b8d0U, + 0xa7f45150U, 0x65417e53U, 0xa4171ac3U, 0x5e273a96U, + 0x6bab3bcbU, 0x459d1ff1U, 0x58faacabU, 0x03e34b93U, + 0xfa302055U, 0x6d76adf6U, 0x76cc8891U, 0x4c02f525U, + 0xd7e54ffcU, 0xcb2ac5d7U, 0x44352680U, 0xa362b58fU, + 0x5ab1de49U, 0x1bba2567U, 0x0eea4598U, 0xc0fe5de1U, + 0x752fc302U, 0xf04c8112U, 0x97468da3U, 0xf9d36bc6U, + 0x5f8f03e7U, 0x9c921595U, 0x7a6dbfebU, 0x595295daU, + 0x83bed42dU, 0x217458d3U, 0x69e04929U, 0xc8c98e44U, + 0x89c2756aU, 0x798ef478U, 0x3e58996bU, 0x71b927ddU, + 0x4fe1beb6U, 0xad88f017U, 0xac20c966U, 0x3ace7db4U, + 0x4adf6318U, 0x311ae582U, 0x33519760U, 0x7f536245U, + 0x7764b1e0U, 0xae6bbb84U, 0xa081fe1cU, 0x2b08f994U, + 0x68487058U, 0xfd458f19U, 0x6cde9487U, 0xf87b52b7U, + 0xd373ab23U, 0x024b72e2U, 0x8f1fe357U, 0xab55662aU, + 0x28ebb207U, 0xc2b52f03U, 0x7bc5869aU, 0x0837d3a5U, + 0x872830f2U, 0xa5bf23b2U, 0x6a0302baU, 0x8216ed5cU, + 0x1ccf8a2bU, 0xb479a792U, 0xf207f3f0U, 0xe2694ea1U, + 0xf4da65cdU, 0xbe0506d5U, 0x6234d11fU, 0xfea6c48aU, + 0x532e349dU, 0x55f3a2a0U, 0xe18a0532U, 0xebf6a475U, + 0xec830b39U, 0xef6040aaU, 0x9f715e06U, 0x106ebd51U, + 0x8a213ef9U, 0x06dd963dU, 0x053eddaeU, 0xbde64d46U, + 0x8d5491b5U, 0x5dc47105U, 0xd406046fU, 0x155060ffU, + 0xfb981924U, 0xe9bdd697U, 0x434089ccU, 0x9ed96777U, + 0x42e8b0bdU, 0x8b890788U, 0x5b19e738U, 0xeec879dbU, + 0x0a7ca147U, 0x0f427ce9U, 0x1e84f8c9U, 0x00000000U, + 0x86800983U, 0xed2b3248U, 0x70111eacU, 0x725a6c4eU, + 0xff0efdfbU, 0x38850f56U, 0xd5ae3d1eU, 0x392d3627U, + 0xd90f0a64U, 0xa65c6821U, 0x545b9bd1U, 0x2e36243aU, + 0x670a0cb1U, 0xe757930fU, 0x96eeb4d2U, 0x919b1b9eU, + 0xc5c0804fU, 0x20dc61a2U, 0x4b775a69U, 0x1a121c16U, + 0xba93e20aU, 0x2aa0c0e5U, 0xe0223c43U, 0x171b121dU, + 0x0d090e0bU, 0xc78bf2adU, 0xa8b62db9U, 0xa91e14c8U, + 0x19f15785U, 0x0775af4cU, 0xdd99eebbU, 0x607fa3fdU, + 0x2601f79fU, 0xf5725cbcU, 0x3b6644c5U, 0x7efb5b34U, + 0x29438b76U, 0xc623cbdcU, 0xfcedb668U, 0xf1e4b863U, + 0xdc31d7caU, 0x85634210U, 0x22971340U, 0x11c68420U, + 0x244a857dU, 0x3dbbd2f8U, 0x32f9ae11U, 0xa129c76dU, + 0x2f9e1d4bU, 0x30b2dcf3U, 0x52860decU, 0xe3c177d0U, + 0x16b32b6cU, 0xb970a999U, 0x489411faU, 0x64e94722U, + 0x8cfca8c4U, 0x3ff0a01aU, 0x2c7d56d8U, 0x903322efU, + 0x4e4987c7U, 0xd138d9c1U, 0xa2ca8cfeU, 0x0bd49836U, + 0x81f5a6cfU, 0xde7aa528U, 0x8eb7da26U, 0xbfad3fa4U, + 0x9d3a2ce4U, 0x9278500dU, 0xcc5f6a9bU, 0x467e5462U, + 0x138df6c2U, 0xb8d890e8U, 0xf7392e5eU, 0xafc382f5U, + 0x805d9fbeU, 0x93d0697cU, 0x2dd56fa9U, 0x1225cfb3U, + 0x99acc83bU, 0x7d1810a7U, 0x639ce86eU, 0xbb3bdb7bU, + 0x7826cd09U, 0x18596ef4U, 0xb79aec01U, 0x9a4f83a8U, + 0x6e95e665U, 0xe6ffaa7eU, 0xcfbc2108U, 0xe815efe6U, + 0x9be7bad9U, 0x366f4aceU, 0x099fead4U, 0x7cb029d6U, + 0xb2a431afU, 0x233f2a31U, 0x94a5c630U, 0x66a235c0U, + 0xbc4e7437U, 0xca82fca6U, 0xd090e0b0U, 0xd8a73315U, + 0x9804f14aU, 0xdaec41f7U, 0x50cd7f0eU, 0xf691172fU, + 0xd64d768dU, 0xb0ef434dU, 0x4daacc54U, 0x0496e4dfU, + 0xb5d19ee3U, 0x886a4c1bU, 0x1f2cc1b8U, 0x5165467fU, + 0xea5e9d04U, 0x358c015dU, 0x7487fa73U, 0x410bfb2eU, + 0x1d67b35aU, 0xd2db9252U, 0x5610e933U, 0x47d66d13U, + 0x61d79a8cU, 0x0ca1377aU, 0x14f8598eU, 0x3c13eb89U, + 0x27a9ceeeU, 0xc961b735U, 0xe51ce1edU, 0xb1477a3cU, + 0xdfd29c59U, 0x73f2553fU, 0xce141879U, 0x37c773bfU, + 0xcdf753eaU, 0xaafd5f5bU, 0x6f3ddf14U, 0xdb447886U, + 0xf3afca81U, 0xc468b93eU, 0x3424382cU, 0x40a3c25fU, + 0xc31d1672U, 0x25e2bc0cU, 0x493c288bU, 0x950dff41U, + 0x01a83971U, 0xb30c08deU, 0xe4b4d89cU, 0xc1566490U, + 0x84cb7b61U, 0xb632d570U, 0x5c6c4874U, 0x57b8d042U, + 0xf45150a7U, 0x417e5365U, 0x171ac3a4U, 0x273a965eU, + 0xab3bcb6bU, 0x9d1ff145U, 0xfaacab58U, 0xe34b9303U, + 0x302055faU, 0x76adf66dU, 0xcc889176U, 0x02f5254cU, + 0xe54ffcd7U, 0x2ac5d7cbU, 0x35268044U, 0x62b58fa3U, + 0xb1de495aU, 0xba25671bU, 0xea45980eU, 0xfe5de1c0U, + 0x2fc30275U, 0x4c8112f0U, 0x468da397U, 0xd36bc6f9U, + 0x8f03e75fU, 0x9215959cU, 0x6dbfeb7aU, 0x5295da59U, + 0xbed42d83U, 0x7458d321U, 0xe0492969U, 0xc98e44c8U, + 0xc2756a89U, 0x8ef47879U, 0x58996b3eU, 0xb927dd71U, + 0xe1beb64fU, 0x88f017adU, 0x20c966acU, 0xce7db43aU, + 0xdf63184aU, 0x1ae58231U, 0x51976033U, 0x5362457fU, + 0x64b1e077U, 0x6bbb84aeU, 0x81fe1ca0U, 0x08f9942bU, + 0x48705868U, 0x458f19fdU, 0xde94876cU, 0x7b52b7f8U, + 0x73ab23d3U, 0x4b72e202U, 0x1fe3578fU, 0x55662aabU, + 0xebb20728U, 0xb52f03c2U, 0xc5869a7bU, 0x37d3a508U, + 0x2830f287U, 0xbf23b2a5U, 0x0302ba6aU, 0x16ed5c82U, + 0xcf8a2b1cU, 0x79a792b4U, 0x07f3f0f2U, 0x694ea1e2U, + 0xda65cdf4U, 0x0506d5beU, 0x34d11f62U, 0xa6c48afeU, + 0x2e349d53U, 0xf3a2a055U, 0x8a0532e1U, 0xf6a475ebU, + 0x830b39ecU, 0x6040aaefU, 0x715e069fU, 0x6ebd5110U, + 0x213ef98aU, 0xdd963d06U, 0x3eddae05U, 0xe64d46bdU, + 0x5491b58dU, 0xc471055dU, 0x06046fd4U, 0x5060ff15U, + 0x981924fbU, 0xbdd697e9U, 0x4089cc43U, 0xd967779eU, + 0xe8b0bd42U, 0x8907888bU, 0x19e7385bU, 0xc879dbeeU, + 0x7ca1470aU, 0x427ce90fU, 0x84f8c91eU, 0x00000000U, + 0x80098386U, 0x2b3248edU, 0x111eac70U, 0x5a6c4e72U, + 0x0efdfbffU, 0x850f5638U, 0xae3d1ed5U, 0x2d362739U, + 0x0f0a64d9U, 0x5c6821a6U, 0x5b9bd154U, 0x36243a2eU, + 0x0a0cb167U, 0x57930fe7U, 0xeeb4d296U, 0x9b1b9e91U, + 0xc0804fc5U, 0xdc61a220U, 0x775a694bU, 0x121c161aU, + 0x93e20abaU, 0xa0c0e52aU, 0x223c43e0U, 0x1b121d17U, + 0x090e0b0dU, 0x8bf2adc7U, 0xb62db9a8U, 0x1e14c8a9U, + 0xf1578519U, 0x75af4c07U, 0x99eebbddU, 0x7fa3fd60U, + 0x01f79f26U, 0x725cbcf5U, 0x6644c53bU, 0xfb5b347eU, + 0x438b7629U, 0x23cbdcc6U, 0xedb668fcU, 0xe4b863f1U, + 0x31d7cadcU, 0x63421085U, 0x97134022U, 0xc6842011U, + 0x4a857d24U, 0xbbd2f83dU, 0xf9ae1132U, 0x29c76da1U, + 0x9e1d4b2fU, 0xb2dcf330U, 0x860dec52U, 0xc177d0e3U, + 0xb32b6c16U, 0x70a999b9U, 0x9411fa48U, 0xe9472264U, + 0xfca8c48cU, 0xf0a01a3fU, 0x7d56d82cU, 0x3322ef90U, + 0x4987c74eU, 0x38d9c1d1U, 0xca8cfea2U, 0xd498360bU, + 0xf5a6cf81U, 0x7aa528deU, 0xb7da268eU, 0xad3fa4bfU, + 0x3a2ce49dU, 0x78500d92U, 0x5f6a9bccU, 0x7e546246U, + 0x8df6c213U, 0xd890e8b8U, 0x392e5ef7U, 0xc382f5afU, + 0x5d9fbe80U, 0xd0697c93U, 0xd56fa92dU, 0x25cfb312U, + 0xacc83b99U, 0x1810a77dU, 0x9ce86e63U, 0x3bdb7bbbU, + 0x26cd0978U, 0x596ef418U, 0x9aec01b7U, 0x4f83a89aU, + 0x95e6656eU, 0xffaa7ee6U, 0xbc2108cfU, 0x15efe6e8U, + 0xe7bad99bU, 0x6f4ace36U, 0x9fead409U, 0xb029d67cU, + 0xa431afb2U, 0x3f2a3123U, 0xa5c63094U, 0xa235c066U, + 0x4e7437bcU, 0x82fca6caU, 0x90e0b0d0U, 0xa73315d8U, + 0x04f14a98U, 0xec41f7daU, 0xcd7f0e50U, 0x91172ff6U, + 0x4d768dd6U, 0xef434db0U, 0xaacc544dU, 0x96e4df04U, + 0xd19ee3b5U, 0x6a4c1b88U, 0x2cc1b81fU, 0x65467f51U, + 0x5e9d04eaU, 0x8c015d35U, 0x87fa7374U, 0x0bfb2e41U, + 0x67b35a1dU, 0xdb9252d2U, 0x10e93356U, 0xd66d1347U, + 0xd79a8c61U, 0xa1377a0cU, 0xf8598e14U, 0x13eb893cU, + 0xa9ceee27U, 0x61b735c9U, 0x1ce1ede5U, 0x477a3cb1U, + 0xd29c59dfU, 0xf2553f73U, 0x141879ceU, 0xc773bf37U, + 0xf753eacdU, 0xfd5f5baaU, 0x3ddf146fU, 0x447886dbU, + 0xafca81f3U, 0x68b93ec4U, 0x24382c34U, 0xa3c25f40U, + 0x1d1672c3U, 0xe2bc0c25U, 0x3c288b49U, 0x0dff4195U, + 0xa8397101U, 0x0c08deb3U, 0xb4d89ce4U, 0x566490c1U, + 0xcb7b6184U, 0x32d570b6U, 0x6c48745cU, 0xb8d04257U, + 0x5150a7f4U, 0x7e536541U, 0x1ac3a417U, 0x3a965e27U, + 0x3bcb6babU, 0x1ff1459dU, 0xacab58faU, 0x4b9303e3U, + 0x2055fa30U, 0xadf66d76U, 0x889176ccU, 0xf5254c02U, + 0x4ffcd7e5U, 0xc5d7cb2aU, 0x26804435U, 0xb58fa362U, + 0xde495ab1U, 0x25671bbaU, 0x45980eeaU, 0x5de1c0feU, + 0xc302752fU, 0x8112f04cU, 0x8da39746U, 0x6bc6f9d3U, + 0x03e75f8fU, 0x15959c92U, 0xbfeb7a6dU, 0x95da5952U, + 0xd42d83beU, 0x58d32174U, 0x492969e0U, 0x8e44c8c9U, + 0x756a89c2U, 0xf478798eU, 0x996b3e58U, 0x27dd71b9U, + 0xbeb64fe1U, 0xf017ad88U, 0xc966ac20U, 0x7db43aceU, + 0x63184adfU, 0xe582311aU, 0x97603351U, 0x62457f53U, + 0xb1e07764U, 0xbb84ae6bU, 0xfe1ca081U, 0xf9942b08U, + 0x70586848U, 0x8f19fd45U, 0x94876cdeU, 0x52b7f87bU, + 0xab23d373U, 0x72e2024bU, 0xe3578f1fU, 0x662aab55U, + 0xb20728ebU, 0x2f03c2b5U, 0x869a7bc5U, 0xd3a50837U, + 0x30f28728U, 0x23b2a5bfU, 0x02ba6a03U, 0xed5c8216U, + 0x8a2b1ccfU, 0xa792b479U, 0xf3f0f207U, 0x4ea1e269U, + 0x65cdf4daU, 0x06d5be05U, 0xd11f6234U, 0xc48afea6U, + 0x349d532eU, 0xa2a055f3U, 0x0532e18aU, 0xa475ebf6U, + 0x0b39ec83U, 0x40aaef60U, 0x5e069f71U, 0xbd51106eU, + 0x3ef98a21U, 0x963d06ddU, 0xddae053eU, 0x4d46bde6U, + 0x91b58d54U, 0x71055dc4U, 0x046fd406U, 0x60ff1550U, + 0x1924fb98U, 0xd697e9bdU, 0x89cc4340U, 0x67779ed9U, + 0xb0bd42e8U, 0x07888b89U, 0xe7385b19U, 0x79dbeec8U, + 0xa1470a7cU, 0x7ce90f42U, 0xf8c91e84U, 0x00000000U, + 0x09838680U, 0x3248ed2bU, 0x1eac7011U, 0x6c4e725aU, + 0xfdfbff0eU, 0x0f563885U, 0x3d1ed5aeU, 0x3627392dU, + 0x0a64d90fU, 0x6821a65cU, 0x9bd1545bU, 0x243a2e36U, + 0x0cb1670aU, 0x930fe757U, 0xb4d296eeU, 0x1b9e919bU, + 0x804fc5c0U, 0x61a220dcU, 0x5a694b77U, 0x1c161a12U, + 0xe20aba93U, 0xc0e52aa0U, 0x3c43e022U, 0x121d171bU, + 0x0e0b0d09U, 0xf2adc78bU, 0x2db9a8b6U, 0x14c8a91eU, + 0x578519f1U, 0xaf4c0775U, 0xeebbdd99U, 0xa3fd607fU, + 0xf79f2601U, 0x5cbcf572U, 0x44c53b66U, 0x5b347efbU, + 0x8b762943U, 0xcbdcc623U, 0xb668fcedU, 0xb863f1e4U, + 0xd7cadc31U, 0x42108563U, 0x13402297U, 0x842011c6U, + 0x857d244aU, 0xd2f83dbbU, 0xae1132f9U, 0xc76da129U, + 0x1d4b2f9eU, 0xdcf330b2U, 0x0dec5286U, 0x77d0e3c1U, + 0x2b6c16b3U, 0xa999b970U, 0x11fa4894U, 0x472264e9U, + 0xa8c48cfcU, 0xa01a3ff0U, 0x56d82c7dU, 0x22ef9033U, + 0x87c74e49U, 0xd9c1d138U, 0x8cfea2caU, 0x98360bd4U, + 0xa6cf81f5U, 0xa528de7aU, 0xda268eb7U, 0x3fa4bfadU, + 0x2ce49d3aU, 0x500d9278U, 0x6a9bcc5fU, 0x5462467eU, + 0xf6c2138dU, 0x90e8b8d8U, 0x2e5ef739U, 0x82f5afc3U, + 0x9fbe805dU, 0x697c93d0U, 0x6fa92dd5U, 0xcfb31225U, + 0xc83b99acU, 0x10a77d18U, 0xe86e639cU, 0xdb7bbb3bU, + 0xcd097826U, 0x6ef41859U, 0xec01b79aU, 0x83a89a4fU, + 0xe6656e95U, 0xaa7ee6ffU, 0x2108cfbcU, 0xefe6e815U, + 0xbad99be7U, 0x4ace366fU, 0xead4099fU, 0x29d67cb0U, + 0x31afb2a4U, 0x2a31233fU, 0xc63094a5U, 0x35c066a2U, + 0x7437bc4eU, 0xfca6ca82U, 0xe0b0d090U, 0x3315d8a7U, + 0xf14a9804U, 0x41f7daecU, 0x7f0e50cdU, 0x172ff691U, + 0x768dd64dU, 0x434db0efU, 0xcc544daaU, 0xe4df0496U, + 0x9ee3b5d1U, 0x4c1b886aU, 0xc1b81f2cU, 0x467f5165U, + 0x9d04ea5eU, 0x015d358cU, 0xfa737487U, 0xfb2e410bU, + 0xb35a1d67U, 0x9252d2dbU, 0xe9335610U, 0x6d1347d6U, + 0x9a8c61d7U, 0x377a0ca1U, 0x598e14f8U, 0xeb893c13U, + 0xceee27a9U, 0xb735c961U, 0xe1ede51cU, 0x7a3cb147U, + 0x9c59dfd2U, 0x553f73f2U, 0x1879ce14U, 0x73bf37c7U, + 0x53eacdf7U, 0x5f5baafdU, 0xdf146f3dU, 0x7886db44U, + 0xca81f3afU, 0xb93ec468U, 0x382c3424U, 0xc25f40a3U, + 0x1672c31dU, 0xbc0c25e2U, 0x288b493cU, 0xff41950dU, + 0x397101a8U, 0x08deb30cU, 0xd89ce4b4U, 0x6490c156U, + 0x7b6184cbU, 0xd570b632U, 0x48745c6cU, 0xd04257b8U, +}; + +)===" +R"===( + +__constant static const uint AES_KEY_FILL[16] = { + 0x6daca553, 0x62716609, 0xdbb5552b, 0xb4f44917, + 0x6d7caf07, 0x846a710d, 0x1725d378, 0x0da1dc4e, + 0x3f1262f1, 0x9f947ec6, 0xf4c0794f, 0x3e20e345, + 0x6aef8135, 0xb1ba317c, 0x16314c88, 0x49169154, +}; + +__constant static const uint AES_STATE_HASH[16] = { + 0x92b52c0d, 0x9fa856de, 0xcc82db47, 0xd7983aad, + 0x338d996e, 0x15c7b798, 0xf59e125a, 0xace78057, + 0x6a770017, 0xae62c7d0, 0x5079506b, 0xe8a07ce4, + 0x630a240c, 0x07ad828d, 0x79a10005, 0x7e994948, +}; + +uint get_byte32(uint a, uint start_bit) { return (a >> start_bit) & 0xFF; } + +#define fillAes_name fillAes1Rx4_scratchpad +#define outputSize RANDOMX_SCRATCHPAD_L3 +#define outputSize0 (outputSize + 64) +#define unroll_factor 8 +#define num_rounds 1 + #include "fillAes1Rx4.cl" +#undef num_rounds +#undef unroll_factor +#undef outputSize +#undef outputSize0 +#undef fillAes_name + +#define fillAes_name fillAes4Rx4_entropy +#define outputSize ENTROPY_SIZE +#define outputSize0 outputSize +#define unroll_factor 2 +#define num_rounds 4 + #include "fillAes1Rx4.cl" +#undef num_rounds +#undef unroll_factor +#undef outputSize +#undef outputSize0 +#undef fillAes_name + +#define inputSize RANDOMX_SCRATCHPAD_L3 + +__attribute__((reqd_work_group_size(64, 1, 1))) +__kernel void hashAes1Rx4(__global const void* input, __global void* hash, uint hashOffsetBytes, uint hashStrideBytes, uint batch_size) +{ + __local uint T[2048]; + + const uint global_index = get_global_id(0); + if (global_index >= batch_size * 4) + return; + + const uint idx = global_index / 4; + const uint sub = global_index % 4; + + for (uint i = get_local_id(0), step = get_local_size(0); i < 2048; i += step) + T[i] = AES_TABLE[i]; + + barrier(CLK_LOCAL_MEM_FENCE); + + uint x[4] = { AES_STATE_HASH[sub * 4], AES_STATE_HASH[sub * 4 + 1], AES_STATE_HASH[sub * 4 + 2], AES_STATE_HASH[sub * 4 + 3] }; + + const uint s1 = ((sub & 1) == 0) ? 8 : 24; + const uint s3 = ((sub & 1) == 0) ? 24 : 8; + + __global const uint4* p = ((__global uint4*) input) + idx * ((inputSize + 64) / sizeof(uint4)) + sub; + + __local const uint* const t0 = ((sub & 1) == 0) ? T : (T + 1024); + __local const uint* const t1 = ((sub & 1) == 0) ? (T + 256) : (T + 1792); + __local const uint* const t2 = ((sub & 1) == 0) ? (T + 512) : (T + 1536); + __local const uint* const t3 = ((sub & 1) == 0) ? (T + 768) : (T + 1280); + + #pragma unroll(8) + for (uint i = 0; i < inputSize / sizeof(uint4); i += 4, p += 4) + { + uint k[4], y[4]; + *(uint4*)(k) = *p; + y[0] = t0[get_byte32(x[0], 0)] ^ t1[get_byte32(x[1], s1)] ^ t2[get_byte32(x[2], 16)] ^ t3[get_byte32(x[3], s3)] ^ k[0]; + y[1] = t0[get_byte32(x[1], 0)] ^ t1[get_byte32(x[2], s1)] ^ t2[get_byte32(x[3], 16)] ^ t3[get_byte32(x[0], s3)] ^ k[1]; + y[2] = t0[get_byte32(x[2], 0)] ^ t1[get_byte32(x[3], s1)] ^ t2[get_byte32(x[0], 16)] ^ t3[get_byte32(x[1], s3)] ^ k[2]; + y[3] = t0[get_byte32(x[3], 0)] ^ t1[get_byte32(x[0], s1)] ^ t2[get_byte32(x[1], 16)] ^ t3[get_byte32(x[2], s3)] ^ k[3]; + x[0] = y[0]; + x[1] = y[1]; + x[2] = y[2]; + x[3] = y[3]; + } + + uint y[4]; + + y[0] = t0[get_byte32(x[0], 0)] ^ t1[get_byte32(x[1], s1)] ^ t2[get_byte32(x[2], 16)] ^ t3[get_byte32(x[3], s3)] ^ 0xf6fa8389; + y[1] = t0[get_byte32(x[1], 0)] ^ t1[get_byte32(x[2], s1)] ^ t2[get_byte32(x[3], 16)] ^ t3[get_byte32(x[0], s3)] ^ 0x8b24949f; + y[2] = t0[get_byte32(x[2], 0)] ^ t1[get_byte32(x[3], s1)] ^ t2[get_byte32(x[0], 16)] ^ t3[get_byte32(x[1], s3)] ^ 0x90dc56bf; + y[3] = t0[get_byte32(x[3], 0)] ^ t1[get_byte32(x[0], s1)] ^ t2[get_byte32(x[1], 16)] ^ t3[get_byte32(x[2], s3)] ^ 0x06890201; + + x[0] = t0[get_byte32(y[0], 0)] ^ t1[get_byte32(y[1], s1)] ^ t2[get_byte32(y[2], 16)] ^ t3[get_byte32(y[3], s3)] ^ 0x61b263d1; + x[1] = t0[get_byte32(y[1], 0)] ^ t1[get_byte32(y[2], s1)] ^ t2[get_byte32(y[3], 16)] ^ t3[get_byte32(y[0], s3)] ^ 0x51f4e03c; + x[2] = t0[get_byte32(y[2], 0)] ^ t1[get_byte32(y[3], s1)] ^ t2[get_byte32(y[0], 16)] ^ t3[get_byte32(y[1], s3)] ^ 0xee1043c6; + x[3] = t0[get_byte32(y[3], 0)] ^ t1[get_byte32(y[0], s1)] ^ t2[get_byte32(y[1], 16)] ^ t3[get_byte32(y[2], s3)] ^ 0xed18f99b; + + *((__global uint4*)(hash) + idx * (hashStrideBytes / sizeof(uint4)) + sub + (hashOffsetBytes / sizeof(uint4))) = *(uint4*)(x); +} +)===" diff --git a/src/backend/opencl/cl/rx/blake2b.cl b/src/backend/opencl/cl/rx/blake2b.cl new file mode 100644 index 00000000..2fbb9564 --- /dev/null +++ b/src/backend/opencl/cl/rx/blake2b.cl @@ -0,0 +1,159 @@ +R"===( +/* +Copyright (c) 2019 SChernykh + +This file is part of RandomX OpenCL. + +RandomX OpenCL is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +RandomX OpenCL is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with RandomX OpenCL. If not, see . +*/ + +__constant static const uchar blake2b_sigma[12 * 16] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3, + 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4, + 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8, + 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13, + 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9, + 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11, + 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10, + 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5, + 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3, +}; + +enum Blake2b_IV +{ + iv0 = 0x6a09e667f3bcc908ul, + iv1 = 0xbb67ae8584caa73bul, + iv2 = 0x3c6ef372fe94f82bul, + iv3 = 0xa54ff53a5f1d36f1ul, + iv4 = 0x510e527fade682d1ul, + iv5 = 0x9b05688c2b3e6c1ful, + iv6 = 0x1f83d9abfb41bd6bul, + iv7 = 0x5be0cd19137e2179ul, +}; + +ulong rotr64(ulong a, ulong shift) { return rotate(a, 64 - shift); } + +#define G(r, i, a, b, c, d) \ + do { \ + a = a + b + m[blake2b_sigma[r * 16 + 2 * i + 0]]; \ + d = rotr64(d ^ a, 32); \ + c = c + d; \ + b = rotr64(b ^ c, 24); \ + a = a + b + m[blake2b_sigma[r * 16 + 2 * i + 1]]; \ + d = rotr64(d ^ a, 16); \ + c = c + d; \ + b = rotr64(b ^ c, 63); \ + } while (0) + +#define ROUND(r) \ + do { \ + G(r, 0, v[0], v[4], v[8], v[12]); \ + G(r, 1, v[1], v[5], v[9], v[13]); \ + G(r, 2, v[2], v[6], v[10], v[14]); \ + G(r, 3, v[3], v[7], v[11], v[15]); \ + G(r, 4, v[0], v[5], v[10], v[15]); \ + G(r, 5, v[1], v[6], v[11], v[12]); \ + G(r, 6, v[2], v[7], v[8], v[13]); \ + G(r, 7, v[3], v[4], v[9], v[14]); \ + } while (0) + +#define BLAKE2B_ROUNDS() ROUND(0);ROUND(1);ROUND(2);ROUND(3);ROUND(4);ROUND(5);ROUND(6);ROUND(7);ROUND(8);ROUND(9);ROUND(10);ROUND(11); + +void blake2b_512_process_single_block(ulong *h, const ulong* m, uint blockTemplateSize) +{ + ulong v[16] = + { + iv0 ^ 0x01010040, iv1, iv2, iv3, iv4 , iv5, iv6, iv7, + iv0 , iv1, iv2, iv3, iv4 ^ blockTemplateSize, iv5, ~iv6, iv7, + }; + + BLAKE2B_ROUNDS(); + + h[0] = v[0] ^ v[ 8] ^ iv0 ^ 0x01010040; + h[1] = v[1] ^ v[ 9] ^ iv1; + h[2] = v[2] ^ v[10] ^ iv2; + h[3] = v[3] ^ v[11] ^ iv3; + h[4] = v[4] ^ v[12] ^ iv4; + h[5] = v[5] ^ v[13] ^ iv5; + h[6] = v[6] ^ v[14] ^ iv6; + h[7] = v[7] ^ v[15] ^ iv7; +} + +__attribute__((reqd_work_group_size(64, 1, 1))) +__kernel void blake2b_initial_hash(__global void *out, __global const void* blockTemplate, uint blockTemplateSize, uint start_nonce) +{ + const uint global_index = get_global_id(0); + + __global const ulong* p = (__global const ulong*) blockTemplate; + ulong m[16] = { + (blockTemplateSize > 0) ? p[ 0] : 0, + (blockTemplateSize > 8) ? p[ 1] : 0, + (blockTemplateSize > 16) ? p[ 2] : 0, + (blockTemplateSize > 24) ? p[ 3] : 0, + (blockTemplateSize > 32) ? p[ 4] : 0, + (blockTemplateSize > 40) ? p[ 5] : 0, + (blockTemplateSize > 48) ? p[ 6] : 0, + (blockTemplateSize > 56) ? p[ 7] : 0, + (blockTemplateSize > 64) ? p[ 8] : 0, + (blockTemplateSize > 72) ? p[ 9] : 0, + (blockTemplateSize > 80) ? p[10] : 0, + (blockTemplateSize > 88) ? p[11] : 0, + (blockTemplateSize > 96) ? p[12] : 0, + (blockTemplateSize > 104) ? p[13] : 0, + (blockTemplateSize > 112) ? p[14] : 0, + (blockTemplateSize > 120) ? p[15] : 0, + }; + + if (blockTemplateSize % sizeof(ulong)) + m[blockTemplateSize / sizeof(ulong)] &= (ulong)(-1) >> (64 - (blockTemplateSize % sizeof(ulong)) * 8); + + const ulong nonce = start_nonce + global_index; + m[4] = (m[4] & ((ulong)(-1) >> 8)) | (nonce << 56); + m[5] = (m[5] & ((ulong)(-1) << 24)) | (nonce >> 8); + + ulong hash[8]; + blake2b_512_process_single_block(hash, m, blockTemplateSize); + + __global ulong* t = ((__global ulong*) out) + global_index * 8; + t[0] = hash[0]; + t[1] = hash[1]; + t[2] = hash[2]; + t[3] = hash[3]; + t[4] = hash[4]; + t[5] = hash[5]; + t[6] = hash[6]; + t[7] = hash[7]; +} + +#define in_len 256 + +#define out_len 32 +#define blake2b_512_process_double_block_name blake2b_512_process_double_block_32 +#define blake2b_hash_registers_name blake2b_hash_registers_32 + #include "blake2b_double_block.cl" +#undef blake2b_hash_registers_name +#undef blake2b_512_process_double_block_name +#undef out_len + +#define out_len 64 +#define blake2b_512_process_double_block_name blake2b_512_process_double_block_64 +#define blake2b_hash_registers_name blake2b_hash_registers_64 + #include "blake2b_double_block.cl" +#undef blake2b_hash_registers_name +#undef blake2b_512_process_double_block_name +#undef out_len +)===" diff --git a/src/backend/opencl/cl/rx/blake2b_double_block.cl b/src/backend/opencl/cl/rx/blake2b_double_block.cl new file mode 100644 index 00000000..36d3377f --- /dev/null +++ b/src/backend/opencl/cl/rx/blake2b_double_block.cl @@ -0,0 +1,102 @@ +R"===( +/* +Copyright (c) 2019 SChernykh + +This file is part of RandomX OpenCL. + +RandomX OpenCL is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +RandomX OpenCL is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with RandomX OpenCL. If not, see . +*/ + +void blake2b_512_process_double_block_name(ulong *out, ulong* m, __global const ulong* in) +{ + ulong v[16] = + { + iv0 ^ (0x01010000u | out_len), iv1, iv2, iv3, iv4 , iv5, iv6, iv7, + iv0 , iv1, iv2, iv3, iv4 ^ 128, iv5, iv6, iv7, + }; + + BLAKE2B_ROUNDS(); + + ulong h[8]; + v[0] = h[0] = v[0] ^ v[8] ^ iv0 ^ (0x01010000u | out_len); + v[1] = h[1] = v[1] ^ v[9] ^ iv1; + v[2] = h[2] = v[2] ^ v[10] ^ iv2; + v[3] = h[3] = v[3] ^ v[11] ^ iv3; + v[4] = h[4] = v[4] ^ v[12] ^ iv4; + v[5] = h[5] = v[5] ^ v[13] ^ iv5; + v[6] = h[6] = v[6] ^ v[14] ^ iv6; + v[7] = h[7] = v[7] ^ v[15] ^ iv7; + v[8] = iv0; + v[9] = iv1; + v[10] = iv2; + v[11] = iv3; + v[12] = iv4 ^ in_len; + v[13] = iv5; + v[14] = ~iv6; + v[15] = iv7; + + m[ 0] = (in_len > 128) ? in[16] : 0; + m[ 1] = (in_len > 136) ? in[17] : 0; + m[ 2] = (in_len > 144) ? in[18] : 0; + m[ 3] = (in_len > 152) ? in[19] : 0; + m[ 4] = (in_len > 160) ? in[20] : 0; + m[ 5] = (in_len > 168) ? in[21] : 0; + m[ 6] = (in_len > 176) ? in[22] : 0; + m[ 7] = (in_len > 184) ? in[23] : 0; + m[ 8] = (in_len > 192) ? in[24] : 0; + m[ 9] = (in_len > 200) ? in[25] : 0; + m[10] = (in_len > 208) ? in[26] : 0; + m[11] = (in_len > 216) ? in[27] : 0; + m[12] = (in_len > 224) ? in[28] : 0; + m[13] = (in_len > 232) ? in[29] : 0; + m[14] = (in_len > 240) ? in[30] : 0; + m[15] = (in_len > 248) ? in[31] : 0; + + if (in_len % sizeof(ulong)) + m[(in_len - 128) / sizeof(ulong)] &= (ulong)(-1) >> (64 - (in_len % sizeof(ulong)) * 8); + + BLAKE2B_ROUNDS(); + + if (out_len > 0) out[0] = h[0] ^ v[0] ^ v[8]; + if (out_len > 8) out[1] = h[1] ^ v[1] ^ v[9]; + if (out_len > 16) out[2] = h[2] ^ v[2] ^ v[10]; + if (out_len > 24) out[3] = h[3] ^ v[3] ^ v[11]; + if (out_len > 32) out[4] = h[4] ^ v[4] ^ v[12]; + if (out_len > 40) out[5] = h[5] ^ v[5] ^ v[13]; + if (out_len > 48) out[6] = h[6] ^ v[6] ^ v[14]; + if (out_len > 56) out[7] = h[7] ^ v[7] ^ v[15]; +} + +__attribute__((reqd_work_group_size(64, 1, 1))) +__kernel void blake2b_hash_registers_name(__global void *out, __global const void* in, uint inStrideBytes) +{ + const uint global_index = get_global_id(0); + __global const ulong* p = ((__global const ulong*) in) + global_index * (inStrideBytes / sizeof(ulong)); + __global ulong* h = ((__global ulong*) out) + global_index * (out_len / sizeof(ulong)); + + ulong m[16] = { p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15] }; + + ulong hash[8]; + blake2b_512_process_double_block_name(hash, m, p); + + if (out_len > 0) h[0] = hash[0]; + if (out_len > 8) h[1] = hash[1]; + if (out_len > 16) h[2] = hash[2]; + if (out_len > 24) h[3] = hash[3]; + if (out_len > 32) h[4] = hash[4]; + if (out_len > 40) h[5] = hash[5]; + if (out_len > 48) h[6] = hash[6]; + if (out_len > 56) h[7] = hash[7]; +} +)===" diff --git a/src/backend/opencl/cl/rx/fillAes1Rx4.cl b/src/backend/opencl/cl/rx/fillAes1Rx4.cl new file mode 100644 index 00000000..a30742f5 --- /dev/null +++ b/src/backend/opencl/cl/rx/fillAes1Rx4.cl @@ -0,0 +1,120 @@ +R"===( +/* +Copyright (c) 2019 SChernykh + +This file is part of RandomX OpenCL. + +RandomX OpenCL is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +RandomX OpenCL is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with RandomX OpenCL. If not, see . +*/ + +__attribute__((reqd_work_group_size(64, 1, 1))) +__kernel void fillAes_name(__global void* state, __global void* out, uint batch_size, uint rx_version) +{ + __local uint T[2048]; + + const uint global_index = get_global_id(0); + if (global_index >= batch_size * 4) + return; + + const uint idx = global_index / 4; + const uint sub = global_index % 4; + + for (uint i = get_local_id(0), step = get_local_size(0); i < 2048; i += step) + T[i] = AES_TABLE[i]; + + barrier(CLK_LOCAL_MEM_FENCE); + +#if num_rounds != 4 + const uint k[4] = { AES_KEY_FILL[sub * 4], AES_KEY_FILL[sub * 4 + 1], AES_KEY_FILL[sub * 4 + 2], AES_KEY_FILL[sub * 4 + 3] }; +#else + const bool b1 = (rx_version < 104); + const bool b2 = (sub < 2); + + uint k[16]; + k[ 0] = b1 ? 0xf890465du : (b2 ? 0x6421aaddu : 0xb5826f73u); + k[ 1] = b1 ? 0x7ffbe4a6u : (b2 ? 0xd1833ddbu : 0xe3d6a7a6u); + k[ 2] = b1 ? 0x141f82b7u : (b2 ? 0x2f546d2bu : 0x3d518b6du); + k[ 3] = b1 ? 0xcf359e95u : (b2 ? 0x99e5d23fu : 0x229effb4u); + k[ 4] = b1 ? 0x6a55c450u : (b2 ? 0xb20e3450u : 0xc7566bf3u); + k[ 5] = b1 ? 0xfee8278au : (b2 ? 0xb6913f55u : 0x9c10b3d9u); + k[ 6] = b1 ? 0xbd5c5ac3u : (b2 ? 0x06f79d53u : 0xe9024d4eu); + k[ 7] = b1 ? 0x6741ffdcu : (b2 ? 0xa5dfcde5u : 0xb272b7d2u); + k[ 8] = b1 ? 0x114c47a4u : (b2 ? 0x5c3ed904u : 0xf273c9e7u); + k[ 9] = b1 ? 0xd524fde4u : (b2 ? 0x515e7bafu : 0xf765a38bu); + k[10] = b1 ? 0xa7279ad2u : (b2 ? 0x0aa4679fu : 0x2ba9660au); + k[11] = b1 ? 0x3d324aacu : (b2 ? 0x171c02bfu : 0xf63befa7u); + k[12] = b1 ? 0x810c3a2au : (b2 ? 0x85623763u : 0x7a7cd609u); + k[13] = b1 ? 0x99a9aeffu : (b2 ? 0xe78f5d08u : 0x915839deu); + k[14] = b1 ? 0x42d3dbd9u : (b2 ? 0xcd673785u : 0x0c06d1fdu); + k[15] = b1 ? 0x76f6db08u : (b2 ? 0xd8ded291u : 0xc0b0762du); +#endif + + __global uint* s = ((__global uint*) state) + idx * (64 / sizeof(uint)) + sub * (16 / sizeof(uint)); + uint x[4] = { s[0], s[1], s[2], s[3] }; + + const uint s1 = (sub & 1) ? 8 : 24; + const uint s3 = (sub & 1) ? 24 : 8; + + __global uint4* p = ((__global uint4*) out) + idx * (outputSize0 / sizeof(uint4)) + sub; + + const __local uint* const t0 = (sub & 1) ? T : (T + 1024); + const __local uint* const t1 = (sub & 1) ? (T + 256) : (T + 1792); + const __local uint* const t2 = (sub & 1) ? (T + 512) : (T + 1536); + const __local uint* const t3 = (sub & 1) ? (T + 768) : (T + 1280); + + #pragma unroll(unroll_factor) + for (uint i = 0; i < outputSize / sizeof(uint4); i += 4, p += 4) + { + uint y[4]; + +#if num_rounds != 4 + y[0] = t0[get_byte32(x[0], 0)] ^ t1[get_byte32(x[1], s1)] ^ t2[get_byte32(x[2], 16)] ^ t3[get_byte32(x[3], s3)] ^ k[0]; + y[1] = t0[get_byte32(x[1], 0)] ^ t1[get_byte32(x[2], s1)] ^ t2[get_byte32(x[3], 16)] ^ t3[get_byte32(x[0], s3)] ^ k[1]; + y[2] = t0[get_byte32(x[2], 0)] ^ t1[get_byte32(x[3], s1)] ^ t2[get_byte32(x[0], 16)] ^ t3[get_byte32(x[1], s3)] ^ k[2]; + y[3] = t0[get_byte32(x[3], 0)] ^ t1[get_byte32(x[0], s1)] ^ t2[get_byte32(x[1], 16)] ^ t3[get_byte32(x[2], s3)] ^ k[3]; + + *p = *(uint4*)(y); + + x[0] = y[0]; + x[1] = y[1]; + x[2] = y[2]; + x[3] = y[3]; +#else + y[0] = t0[get_byte32(x[0], 0)] ^ t1[get_byte32(x[1], s1)] ^ t2[get_byte32(x[2], 16)] ^ t3[get_byte32(x[3], s3)] ^ k[ 0]; + y[1] = t0[get_byte32(x[1], 0)] ^ t1[get_byte32(x[2], s1)] ^ t2[get_byte32(x[3], 16)] ^ t3[get_byte32(x[0], s3)] ^ k[ 1]; + y[2] = t0[get_byte32(x[2], 0)] ^ t1[get_byte32(x[3], s1)] ^ t2[get_byte32(x[0], 16)] ^ t3[get_byte32(x[1], s3)] ^ k[ 2]; + y[3] = t0[get_byte32(x[3], 0)] ^ t1[get_byte32(x[0], s1)] ^ t2[get_byte32(x[1], 16)] ^ t3[get_byte32(x[2], s3)] ^ k[ 3]; + + x[0] = t0[get_byte32(y[0], 0)] ^ t1[get_byte32(y[1], s1)] ^ t2[get_byte32(y[2], 16)] ^ t3[get_byte32(y[3], s3)] ^ k[ 4]; + x[1] = t0[get_byte32(y[1], 0)] ^ t1[get_byte32(y[2], s1)] ^ t2[get_byte32(y[3], 16)] ^ t3[get_byte32(y[0], s3)] ^ k[ 5]; + x[2] = t0[get_byte32(y[2], 0)] ^ t1[get_byte32(y[3], s1)] ^ t2[get_byte32(y[0], 16)] ^ t3[get_byte32(y[1], s3)] ^ k[ 6]; + x[3] = t0[get_byte32(y[3], 0)] ^ t1[get_byte32(y[0], s1)] ^ t2[get_byte32(y[1], 16)] ^ t3[get_byte32(y[2], s3)] ^ k[ 7]; + + y[0] = t0[get_byte32(x[0], 0)] ^ t1[get_byte32(x[1], s1)] ^ t2[get_byte32(x[2], 16)] ^ t3[get_byte32(x[3], s3)] ^ k[ 8]; + y[1] = t0[get_byte32(x[1], 0)] ^ t1[get_byte32(x[2], s1)] ^ t2[get_byte32(x[3], 16)] ^ t3[get_byte32(x[0], s3)] ^ k[ 9]; + y[2] = t0[get_byte32(x[2], 0)] ^ t1[get_byte32(x[3], s1)] ^ t2[get_byte32(x[0], 16)] ^ t3[get_byte32(x[1], s3)] ^ k[10]; + y[3] = t0[get_byte32(x[3], 0)] ^ t1[get_byte32(x[0], s1)] ^ t2[get_byte32(x[1], 16)] ^ t3[get_byte32(x[2], s3)] ^ k[11]; + + x[0] = t0[get_byte32(y[0], 0)] ^ t1[get_byte32(y[1], s1)] ^ t2[get_byte32(y[2], 16)] ^ t3[get_byte32(y[3], s3)] ^ k[12]; + x[1] = t0[get_byte32(y[1], 0)] ^ t1[get_byte32(y[2], s1)] ^ t2[get_byte32(y[3], 16)] ^ t3[get_byte32(y[0], s3)] ^ k[13]; + x[2] = t0[get_byte32(y[2], 0)] ^ t1[get_byte32(y[3], s1)] ^ t2[get_byte32(y[0], 16)] ^ t3[get_byte32(y[1], s3)] ^ k[14]; + x[3] = t0[get_byte32(y[3], 0)] ^ t1[get_byte32(y[0], s1)] ^ t2[get_byte32(y[1], 16)] ^ t3[get_byte32(y[2], s3)] ^ k[15]; + + *p = *(uint4*)(x); +#endif + } + + *(__global uint4*)(s) = *(uint4*)(x); +} +)===" diff --git a/src/backend/opencl/cl/rx/randomx_constants_loki.h b/src/backend/opencl/cl/rx/randomx_constants_loki.h new file mode 100644 index 00000000..065da96c --- /dev/null +++ b/src/backend/opencl/cl/rx/randomx_constants_loki.h @@ -0,0 +1,98 @@ +R"===( +/* +Copyright (c) 2019 SChernykh + +This file is part of RandomX OpenCL. + +RandomX OpenCL is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +RandomX OpenCL is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with RandomX OpenCL. If not, see . +*/ + +//Dataset base size in bytes. Must be a power of 2. +#define RANDOMX_DATASET_BASE_SIZE 2147483648 + +//Dataset extra size. Must be divisible by 64. +#define RANDOMX_DATASET_EXTRA_SIZE 33554368 + +//Scratchpad L3 size in bytes. Must be a power of 2. +#define RANDOMX_SCRATCHPAD_L3 2097152 + +//Scratchpad L2 size in bytes. Must be a power of two and less than or equal to RANDOMX_SCRATCHPAD_L3. +#define RANDOMX_SCRATCHPAD_L2 262144 + +//Scratchpad L1 size in bytes. Must be a power of two (minimum 64) and less than or equal to RANDOMX_SCRATCHPAD_L2. +#define RANDOMX_SCRATCHPAD_L1 16384 + +//Jump condition mask size in bits. +#define RANDOMX_JUMP_BITS 8 + +//Jump condition mask offset in bits. The sum of RANDOMX_JUMP_BITS and RANDOMX_JUMP_OFFSET must not exceed 16. +#define RANDOMX_JUMP_OFFSET 8 + +//Integer instructions +#define RANDOMX_FREQ_IADD_RS 25 +#define RANDOMX_FREQ_IADD_M 7 +#define RANDOMX_FREQ_ISUB_R 16 +#define RANDOMX_FREQ_ISUB_M 7 +#define RANDOMX_FREQ_IMUL_R 16 +#define RANDOMX_FREQ_IMUL_M 4 +#define RANDOMX_FREQ_IMULH_R 4 +#define RANDOMX_FREQ_IMULH_M 1 +#define RANDOMX_FREQ_ISMULH_R 4 +#define RANDOMX_FREQ_ISMULH_M 1 +#define RANDOMX_FREQ_IMUL_RCP 8 +#define RANDOMX_FREQ_INEG_R 2 +#define RANDOMX_FREQ_IXOR_R 15 +#define RANDOMX_FREQ_IXOR_M 5 +#define RANDOMX_FREQ_IROR_R 8 +#define RANDOMX_FREQ_IROL_R 2 +#define RANDOMX_FREQ_ISWAP_R 4 + +//Floating point instructions +#define RANDOMX_FREQ_FSWAP_R 4 +#define RANDOMX_FREQ_FADD_R 16 +#define RANDOMX_FREQ_FADD_M 5 +#define RANDOMX_FREQ_FSUB_R 16 +#define RANDOMX_FREQ_FSUB_M 5 +#define RANDOMX_FREQ_FSCAL_R 6 +#define RANDOMX_FREQ_FMUL_R 32 +#define RANDOMX_FREQ_FDIV_M 4 +#define RANDOMX_FREQ_FSQRT_R 6 + +//Control instructions +#define RANDOMX_FREQ_CBRANCH 16 +#define RANDOMX_FREQ_CFROUND 1 + +//Store instruction +#define RANDOMX_FREQ_ISTORE 16 + +//No-op instruction +#define RANDOMX_FREQ_NOP 0 + +#define RANDOMX_DATASET_ITEM_SIZE 64 + +#define RANDOMX_PROGRAM_SIZE 320 + +#define HASH_SIZE 64 +#define ENTROPY_SIZE (128 + RANDOMX_PROGRAM_SIZE * 8) +#define REGISTERS_SIZE 256 +#define IMM_BUF_SIZE (RANDOMX_PROGRAM_SIZE * 4 - REGISTERS_SIZE) +#define IMM_INDEX_COUNT ((IMM_BUF_SIZE / 4) - 2) +#define VM_STATE_SIZE (REGISTERS_SIZE + IMM_BUF_SIZE + RANDOMX_PROGRAM_SIZE * 4) +#define ROUNDING_MODE (RANDOMX_FREQ_CFROUND ? -1 : 0) + +// Scratchpad L1/L2/L3 bits +#define LOC_L1 (32 - 14) +#define LOC_L2 (32 - 18) +#define LOC_L3 (32 - 21) +)===" diff --git a/src/backend/opencl/cl/rx/randomx_constants_monero.h b/src/backend/opencl/cl/rx/randomx_constants_monero.h new file mode 100644 index 00000000..8cb78883 --- /dev/null +++ b/src/backend/opencl/cl/rx/randomx_constants_monero.h @@ -0,0 +1,98 @@ +R"===( +/* +Copyright (c) 2019 SChernykh + +This file is part of RandomX OpenCL. + +RandomX OpenCL is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +RandomX OpenCL is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with RandomX OpenCL. If not, see . +*/ + +//Dataset base size in bytes. Must be a power of 2. +#define RANDOMX_DATASET_BASE_SIZE 2147483648 + +//Dataset extra size. Must be divisible by 64. +#define RANDOMX_DATASET_EXTRA_SIZE 33554368 + +//Scratchpad L3 size in bytes. Must be a power of 2. +#define RANDOMX_SCRATCHPAD_L3 2097152 + +//Scratchpad L2 size in bytes. Must be a power of two and less than or equal to RANDOMX_SCRATCHPAD_L3. +#define RANDOMX_SCRATCHPAD_L2 262144 + +//Scratchpad L1 size in bytes. Must be a power of two (minimum 64) and less than or equal to RANDOMX_SCRATCHPAD_L2. +#define RANDOMX_SCRATCHPAD_L1 16384 + +//Jump condition mask size in bits. +#define RANDOMX_JUMP_BITS 8 + +//Jump condition mask offset in bits. The sum of RANDOMX_JUMP_BITS and RANDOMX_JUMP_OFFSET must not exceed 16. +#define RANDOMX_JUMP_OFFSET 8 + +//Integer instructions +#define RANDOMX_FREQ_IADD_RS 25 +#define RANDOMX_FREQ_IADD_M 7 +#define RANDOMX_FREQ_ISUB_R 16 +#define RANDOMX_FREQ_ISUB_M 7 +#define RANDOMX_FREQ_IMUL_R 16 +#define RANDOMX_FREQ_IMUL_M 4 +#define RANDOMX_FREQ_IMULH_R 4 +#define RANDOMX_FREQ_IMULH_M 1 +#define RANDOMX_FREQ_ISMULH_R 4 +#define RANDOMX_FREQ_ISMULH_M 1 +#define RANDOMX_FREQ_IMUL_RCP 8 +#define RANDOMX_FREQ_INEG_R 2 +#define RANDOMX_FREQ_IXOR_R 15 +#define RANDOMX_FREQ_IXOR_M 5 +#define RANDOMX_FREQ_IROR_R 8 +#define RANDOMX_FREQ_IROL_R 2 +#define RANDOMX_FREQ_ISWAP_R 4 + +//Floating point instructions +#define RANDOMX_FREQ_FSWAP_R 4 +#define RANDOMX_FREQ_FADD_R 16 +#define RANDOMX_FREQ_FADD_M 5 +#define RANDOMX_FREQ_FSUB_R 16 +#define RANDOMX_FREQ_FSUB_M 5 +#define RANDOMX_FREQ_FSCAL_R 6 +#define RANDOMX_FREQ_FMUL_R 32 +#define RANDOMX_FREQ_FDIV_M 4 +#define RANDOMX_FREQ_FSQRT_R 6 + +//Control instructions +#define RANDOMX_FREQ_CBRANCH 16 +#define RANDOMX_FREQ_CFROUND 1 + +//Store instruction +#define RANDOMX_FREQ_ISTORE 16 + +//No-op instruction +#define RANDOMX_FREQ_NOP 0 + +#define RANDOMX_DATASET_ITEM_SIZE 64 + +#define RANDOMX_PROGRAM_SIZE 256 + +#define HASH_SIZE 64 +#define ENTROPY_SIZE (128 + RANDOMX_PROGRAM_SIZE * 8) +#define REGISTERS_SIZE 256 +#define IMM_BUF_SIZE (RANDOMX_PROGRAM_SIZE * 4 - REGISTERS_SIZE) +#define IMM_INDEX_COUNT ((IMM_BUF_SIZE / 4) - 2) +#define VM_STATE_SIZE (REGISTERS_SIZE + IMM_BUF_SIZE + RANDOMX_PROGRAM_SIZE * 4) +#define ROUNDING_MODE (RANDOMX_FREQ_CFROUND ? -1 : 0) + +// Scratchpad L1/L2/L3 bits +#define LOC_L1 (32 - 14) +#define LOC_L2 (32 - 18) +#define LOC_L3 (32 - 21) +)===" diff --git a/src/backend/opencl/cl/rx/randomx_constants_wow.h b/src/backend/opencl/cl/rx/randomx_constants_wow.h new file mode 100644 index 00000000..60f10671 --- /dev/null +++ b/src/backend/opencl/cl/rx/randomx_constants_wow.h @@ -0,0 +1,98 @@ +R"===( +/* +Copyright (c) 2019 SChernykh + +This file is part of RandomX OpenCL. + +RandomX OpenCL is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +RandomX OpenCL is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with RandomX OpenCL. If not, see . +*/ + +//Dataset base size in bytes. Must be a power of 2. +#define RANDOMX_DATASET_BASE_SIZE 2147483648 + +//Dataset extra size. Must be divisible by 64. +#define RANDOMX_DATASET_EXTRA_SIZE 33554368 + +//Scratchpad L3 size in bytes. Must be a power of 2. +#define RANDOMX_SCRATCHPAD_L3 1048576 + +//Scratchpad L2 size in bytes. Must be a power of two and less than or equal to RANDOMX_SCRATCHPAD_L3. +#define RANDOMX_SCRATCHPAD_L2 131072 + +//Scratchpad L1 size in bytes. Must be a power of two (minimum 64) and less than or equal to RANDOMX_SCRATCHPAD_L2. +#define RANDOMX_SCRATCHPAD_L1 16384 + +//Jump condition mask size in bits. +#define RANDOMX_JUMP_BITS 8 + +//Jump condition mask offset in bits. The sum of RANDOMX_JUMP_BITS and RANDOMX_JUMP_OFFSET must not exceed 16. +#define RANDOMX_JUMP_OFFSET 8 + +//Integer instructions +#define RANDOMX_FREQ_IADD_RS 25 +#define RANDOMX_FREQ_IADD_M 7 +#define RANDOMX_FREQ_ISUB_R 16 +#define RANDOMX_FREQ_ISUB_M 7 +#define RANDOMX_FREQ_IMUL_R 16 +#define RANDOMX_FREQ_IMUL_M 4 +#define RANDOMX_FREQ_IMULH_R 4 +#define RANDOMX_FREQ_IMULH_M 1 +#define RANDOMX_FREQ_ISMULH_R 4 +#define RANDOMX_FREQ_ISMULH_M 1 +#define RANDOMX_FREQ_IMUL_RCP 8 +#define RANDOMX_FREQ_INEG_R 2 +#define RANDOMX_FREQ_IXOR_R 15 +#define RANDOMX_FREQ_IXOR_M 5 +#define RANDOMX_FREQ_IROR_R 10 +#define RANDOMX_FREQ_IROL_R 0 +#define RANDOMX_FREQ_ISWAP_R 4 + +//Floating point instructions +#define RANDOMX_FREQ_FSWAP_R 8 +#define RANDOMX_FREQ_FADD_R 20 +#define RANDOMX_FREQ_FADD_M 5 +#define RANDOMX_FREQ_FSUB_R 20 +#define RANDOMX_FREQ_FSUB_M 5 +#define RANDOMX_FREQ_FSCAL_R 6 +#define RANDOMX_FREQ_FMUL_R 20 +#define RANDOMX_FREQ_FDIV_M 4 +#define RANDOMX_FREQ_FSQRT_R 6 + +//Control instructions +#define RANDOMX_FREQ_CBRANCH 16 +#define RANDOMX_FREQ_CFROUND 1 + +//Store instruction +#define RANDOMX_FREQ_ISTORE 16 + +//No-op instruction +#define RANDOMX_FREQ_NOP 0 + +#define RANDOMX_DATASET_ITEM_SIZE 64 + +#define RANDOMX_PROGRAM_SIZE 256 + +#define HASH_SIZE 64 +#define ENTROPY_SIZE (128 + RANDOMX_PROGRAM_SIZE * 8) +#define REGISTERS_SIZE 256 +#define IMM_BUF_SIZE (RANDOMX_PROGRAM_SIZE * 4 - REGISTERS_SIZE) +#define IMM_INDEX_COUNT ((IMM_BUF_SIZE / 4) - 2) +#define VM_STATE_SIZE (REGISTERS_SIZE + IMM_BUF_SIZE + RANDOMX_PROGRAM_SIZE * 4) +#define ROUNDING_MODE (RANDOMX_FREQ_CFROUND ? -1 : 0) + +// Scratchpad L1/L2/L3 bits +#define LOC_L1 (32 - 14) +#define LOC_L2 (32 - 17) +#define LOC_L3 (32 - 20) +)===" diff --git a/src/backend/opencl/cl/rx/randomx_jit.cl b/src/backend/opencl/cl/rx/randomx_jit.cl new file mode 100644 index 00000000..59d5697d --- /dev/null +++ b/src/backend/opencl/cl/rx/randomx_jit.cl @@ -0,0 +1,1510 @@ +R"===( +/* +Copyright (c) 2019 SChernykh +Portions Copyright (c) 2018-2019 tevador + +This file is part of RandomX OpenCL. + +RandomX OpenCL is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +RandomX OpenCL is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with RandomX OpenCL. If not, see . +*/ + +#define INITIAL_HASH_SIZE 64 +#define INTERMEDIATE_PROGRAM_SIZE (RANDOMX_PROGRAM_SIZE * 16) +#define COMPILED_PROGRAM_SIZE 10048 +#define NUM_VGPR_REGISTERS 128 + +#define mantissaSize 52 +#define exponentSize 11 +#define mantissaMask ((1UL << mantissaSize) - 1) +#define exponentMask ((1UL << exponentSize) - 1) +#define exponentBias 1023 + +#define dynamicExponentBits 4 +#define staticExponentBits 4 +#define constExponentBits 0x300 +#define dynamicMantissaMask ((1UL << (mantissaSize + dynamicExponentBits)) - 1) + +#define ScratchpadL1Mask_reg 38 +#define ScratchpadL2Mask_reg 39 +#define ScratchpadL3Mask_reg 50 + +#define ScratchpadL3Mask (RANDOMX_SCRATCHPAD_L3 - 8) + +#define RANDOMX_JUMP_BITS 8 +#define RANDOMX_JUMP_OFFSET 8 + +__global uint* jit_scratchpad_calc_address(__global uint* p, uint src, uint imm32, uint mask_reg, uint batch_size) +{ + // s_add_i32 s14, s(16 + src * 2), imm32 + *(p++) = 0x810eff10u | (src << 1); + *(p++) = imm32; + + // v_and_b32 v28, s14, mask_reg + *(p++) = 0x2638000eu | (mask_reg << 9); + + return p; +} + +__global uint* jit_scratchpad_calc_fixed_address(__global uint* p, uint imm32, uint batch_size) +{ + // v_mov_b32 v28, imm32 + *(p++) = 0x7e3802ffu; + *(p++) = imm32; + + return p; +} + +__global uint* jit_scratchpad_load(__global uint* p, uint vgpr_index) +{ + // v28 = offset + +#if GCN_VERSION >= 14 + // global_load_dwordx2 v[vgpr_index:vgpr_index+1], v28, s[0:1] + *(p++) = 0xdc548000u; + *(p++) = 0x0000001cu | (vgpr_index << 24); +#else + *(p++) = 0x32543902u; // v_add_u32 v42, vcc, v2, v28 + *(p++) = 0xd11c6a2bu; // v_addc_u32 v43, vcc, v3, 0, vcc + *(p++) = 0x01a90103u; + *(p++) = 0xdc540000u; // flat_load_dwordx2 v[vgpr_index:vgpr_index+1], v[42:43] + *(p++) = 0x0000002au | (vgpr_index << 24); +#endif + + return p; +} + +__global uint* jit_scratchpad_load2(__global uint* p, uint vgpr_index, int vmcnt) +{ + // s_waitcnt vmcnt(N) + if (vmcnt >= 0) + *(p++) = 0xbf8c0f70u | (vmcnt & 15) | ((vmcnt >> 4) << 14); + + // v_readlane_b32 s14, vgpr_index, 0 + *(p++) = 0xd289000eu; + *(p++) = 0x00010100u | vgpr_index; + + // v_readlane_b32 s15, vgpr_index + 1, 0 + *(p++) = 0xd289000fu; + *(p++) = 0x00010100u | (vgpr_index + 1); + + return p; +} + +__global uint* jit_scratchpad_calc_address_fp(__global uint* p, uint src, uint imm32, uint mask_reg, uint batch_size) +{ + // s_add_i32 s14, s(16 + src * 2), imm32 + *(p++) = 0x810eff10u | (src << 1); + *(p++) = imm32; + + // v_and_b32 v28, s14, mask_reg + *(p++) = 0x2638000eu | (mask_reg << 9); + +#if GCN_VERSION >= 14 + // v_add_u32 v28, v28, v44 + *(p++) = 0x6838591cu; +#else + // v_add_u32 v28, vcc, v28, v44 + *(p++) = 0x3238591cu; +#endif + + return p; +} + +__global uint* jit_scratchpad_load_fp(__global uint* p, uint vgpr_index) +{ + // v28 = offset + +#if GCN_VERSION >= 14 + // global_load_dword v(vgpr_index), v28, s[0:1] + *(p++) = 0xdc508000u; + *(p++) = 0x0000001cu | (vgpr_index << 24); +#else + *(p++) = 0x32543902u; // v_add_u32 v42, vcc, v2, v28 + *(p++) = 0xd11c6a2bu; // v_addc_u32 v43, vcc, v3, 0, vcc + *(p++) = 0x01a90103u; + *(p++) = 0xdc500000u; // flat_load_dword v(vgpr_index), v[42:43] + *(p++) = 0x0000002au | (vgpr_index << 24); +#endif + + return p; +} + +__global uint* jit_scratchpad_load2_fp(__global uint* p, uint vgpr_index, int vmcnt) +{ + // s_waitcnt vmcnt(N) + if (vmcnt >= 0) + *(p++) = 0xbf8c0f70u | (vmcnt & 15) | ((vmcnt >> 4) << 14); + + // v_cvt_f64_i32 v[28:29], vgpr_index + *(p++) = 0x7e380900u | vgpr_index; + + return p; +} + +)===" +R"===( + +__global uint* jit_emit_instruction(__global uint* p, __global uint* last_branch_target, const uint2 inst, int prefetch_vgpr_index, int vmcnt, uint batch_size) +{ + uint opcode = inst.x & 0xFF; + const uint dst = (inst.x >> 8) & 7; + const uint src = (inst.x >> 16) & 7; + const uint mod = inst.x >> 24; + + if (opcode < RANDOMX_FREQ_IADD_RS) + { + const uint shift = (mod >> 2) % 4; + if (shift > 0) // p = 3/4 + { + // s_lshl_b64 s[14:15], s[(16 + src * 2):(17 + src * 2)], shift + *(p++) = 0x8e8e8010u | (src << 1) | (shift << 8); + + // s_add_u32 s(16 + dst * 2), s(16 + dst * 2), s14 + *(p++) = 0x80100e10u | (dst << 1) | (dst << 17); + + // s_addc_u32 s(17 + dst * 2), s(17 + dst * 2), s15 + *(p++) = 0x82110f11u | (dst << 1) | (dst << 17); + } + else // p = 1/4 + { + // s_add_u32 s(16 + dst * 2), s(16 + dst * 2), s(16 + src * 2) + *(p++) = 0x80101010u | (dst << 1) | (dst << 17) | (src << 9); + + // s_addc_u32 s(17 + dst * 2), s(17 + dst * 2), s(17 + src * 2) + *(p++) = 0x82111111u | (dst << 1) | (dst << 17) | (src << 9); + } + + if (dst == 5) // p = 1/8 + { + // s_add_u32 s(16 + dst * 2), s(16 + dst * 2), imm32 + *(p++) = 0x8010ff10u | (dst << 1) | (dst << 17); + *(p++) = inst.y; + + // s_addc_u32 s(17 + dst * 2), s(17 + dst * 2), ((inst.y < 0) ? -1 : 0) + *(p++) = 0x82110011u | (dst << 1) | (dst << 17) | (((as_int(inst.y) < 0) ? 0xc1 : 0x80) << 8); + } + + // 12*3/4 + 8*1/4 + 12/8 = 12.5 bytes on average + return p; + } + opcode -= RANDOMX_FREQ_IADD_RS; + + if (opcode < RANDOMX_FREQ_IADD_M) + { + if (prefetch_vgpr_index >= 0) + { + if (src != dst) // p = 7/8 + p = jit_scratchpad_calc_address(p, src, inst.y, (mod % 4) ? ScratchpadL1Mask_reg : ScratchpadL2Mask_reg, batch_size); + else // p = 1/8 + p = jit_scratchpad_calc_fixed_address(p, inst.y & ScratchpadL3Mask, batch_size); + + p = jit_scratchpad_load(p, prefetch_vgpr_index ? prefetch_vgpr_index : 28); + } + + if (prefetch_vgpr_index <= 0) + { + p = jit_scratchpad_load2(p, prefetch_vgpr_index ? -prefetch_vgpr_index : 28, prefetch_vgpr_index ? vmcnt : 0); + + // s_add_u32 s(16 + dst * 2), s(16 + dst * 2), s14 + *(p++) = 0x80100e10u | (dst << 1) | (dst << 17); + + // s_addc_u32 s(17 + dst * 2), s(17 + dst * 2), s15 + *(p++) = 0x82110f11u | (dst << 1) | (dst << 17); + } + + // (12*7/8 + 8*1/8 + 28) + 8 = 47.5 bytes on average + return p; + } + opcode -= RANDOMX_FREQ_IADD_M; + + if (opcode < RANDOMX_FREQ_ISUB_R) + { + if (src != dst) // p = 7/8 + { + // s_sub_u32 s(16 + dst * 2), s(16 + dst * 2), s(16 + src * 2) + *(p++) = 0x80901010u | (dst << 1) | (dst << 17) | (src << 9); + + // s_subb_u32 s(17 + dst * 2), s(17 + dst * 2), s(17 + src * 2) + *(p++) = 0x82911111u | (dst << 1) | (dst << 17) | (src << 9); + } + else // p = 1/8 + { + // s_sub_u32 s(16 + dst * 2), s(16 + dst * 2), imm32 + *(p++) = 0x8090ff10u | (dst << 1) | (dst << 17); + *(p++) = inst.y; + + // s_subb_u32 s(17 + dst * 2), s(17 + dst * 2), ((inst.y < 0) ? -1 : 0) + *(p++) = 0x82910011u | (dst << 1) | (dst << 17) | (((as_int(inst.y) < 0) ? 0xc1 : 0x80) << 8); + } + + // 8*7/8 + 12/8 = 8.5 bytes on average + return p; + } + opcode -= RANDOMX_FREQ_ISUB_R; + + if (opcode < RANDOMX_FREQ_ISUB_M) + { + if (prefetch_vgpr_index >= 0) + { + if (src != dst) // p = 7/8 + p = jit_scratchpad_calc_address(p, src, inst.y, (mod % 4) ? ScratchpadL1Mask_reg : ScratchpadL2Mask_reg, batch_size); + else // p = 1/8 + p = jit_scratchpad_calc_fixed_address(p, inst.y & ScratchpadL3Mask, batch_size); + + p = jit_scratchpad_load(p, prefetch_vgpr_index ? prefetch_vgpr_index : 28); + } + + if (prefetch_vgpr_index <= 0) + { + p = jit_scratchpad_load2(p, prefetch_vgpr_index ? -prefetch_vgpr_index : 28, prefetch_vgpr_index ? vmcnt : 0); + + // s_sub_u32 s(16 + dst * 2), s(16 + dst * 2), s14 + *(p++) = 0x80900e10u | (dst << 1) | (dst << 17); + + // s_subb_u32 s(17 + dst * 2), s(17 + dst * 2), s15 + *(p++) = 0x82910f11u | (dst << 1) | (dst << 17); + } + + // (12*7/8 + 8*1/8 + 28) + 8 = 47.5 bytes on average + return p; + } + opcode -= RANDOMX_FREQ_ISUB_M; + + if (opcode < RANDOMX_FREQ_IMUL_R) + { + if (src != dst) // p = 7/8 + { +#if GCN_VERSION >= 14 + // s_mul_hi_u32 s15, s(16 + dst * 2), s(16 + src * 2) + *(p++) = 0x960f1010u | (dst << 1) | (src << 9); +#else + // v_mov_b32 v28, s(16 + dst * 2) + *(p++) = 0x7e380210u | (dst << 1); + // v_mul_hi_u32 v28, v28, s(16 + src * 2) + *(p++) = 0xd286001cu; + *(p++) = 0x0000211cu + (src << 10); + // v_readlane_b32 s15, v28, 0 + *(p++) = 0xd289000fu; + *(p++) = 0x0001011cu; +#endif + + // s_mul_i32 s14, s(16 + dst * 2), s(17 + src * 2) + *(p++) = 0x920e1110u | (dst << 1) | (src << 9); + + // s_add_u32 s15, s15, s14 + *(p++) = 0x800f0e0fu; + + // s_mul_i32 s14, s(17 + dst * 2), s(16 + src * 2) + *(p++) = 0x920e1011u | (dst << 1) | (src << 9); + + // s_add_u32 s(17 + dst * 2), s15, s14 + *(p++) = 0x80110e0fu | (dst << 17); + + // s_mul_i32 s(16 + dst * 2), s(16 + dst * 2), s(16 + src * 2) + *(p++) = 0x92101010u | (dst << 1) | (dst << 17) | (src << 9); + } + else // p = 1/8 + { +#if GCN_VERSION >= 14 + // s_mul_hi_u32 s15, s(16 + dst * 2), imm32 + *(p++) = 0x960fff10u | (dst << 1); + *(p++) = inst.y; +#else + // v_mov_b32 v28, imm32 + *(p++) = 0x7e3802ffu; + *(p++) = inst.y; + // v_mul_hi_u32 v28, v28, s(16 + dst * 2) + *(p++) = 0xd286001cu; + *(p++) = 0x0000211cu + (dst << 10); + // v_readlane_b32 s15, v28, 0 + *(p++) = 0xd289000fu; + *(p++) = 0x0001011cu; +#endif + + if (as_int(inst.y) < 0) // p = 1/2 + { + // s_sub_u32 s15, s15, s(16 + dst * 2) + *(p++) = 0x808f100fu | (dst << 9); + } + + // s_mul_i32 s14, s(17 + dst * 2), imm32 + *(p++) = 0x920eff11u | (dst << 1); + *(p++) = inst.y; + + // s_add_u32 s(17 + dst * 2), s15, s14 + *(p++) = 0x80110e0fu | (dst << 17); + + // s_mul_i32 s(16 + dst * 2), s(16 + dst * 2), imm32 + *(p++) = 0x9210ff10u | (dst << 1) | (dst << 17); + *(p++) = inst.y; + } + + // 24*7/8 + 28*1/8 + 4*1/16 = 24.75 bytes on average + return p; + } + opcode -= RANDOMX_FREQ_IMUL_R; + + if (opcode < RANDOMX_FREQ_IMUL_M) + { + if (prefetch_vgpr_index >= 0) + { + if (src != dst) // p = 7/8 + p = jit_scratchpad_calc_address(p, src, inst.y, (mod % 4) ? ScratchpadL1Mask_reg : ScratchpadL2Mask_reg, batch_size); + else // p = 1/8 + p = jit_scratchpad_calc_fixed_address(p, inst.y & ScratchpadL3Mask, batch_size); + + p = jit_scratchpad_load(p, prefetch_vgpr_index ? prefetch_vgpr_index : 28); + } + + if (prefetch_vgpr_index <= 0) + { + p = jit_scratchpad_load2(p, prefetch_vgpr_index ? -prefetch_vgpr_index : 28, prefetch_vgpr_index ? vmcnt : 0); + +#if GCN_VERSION >= 14 + // s_mul_hi_u32 s33, s(16 + dst * 2), s14 + *(p++) = 0x96210e10u | (dst << 1); +#else + // v_mov_b32 v28, s(16 + dst * 2) + *(p++) = 0x7e380210u | (dst << 1); + // v_mul_hi_u32 v28, v28, s14 + *(p++) = 0xd286001cu; + *(p++) = 0x00001d1cu; + // v_readlane_b32 s33, v28, 0 + *(p++) = 0xd2890021u; + *(p++) = 0x0001011cu; +#endif + + // s_mul_i32 s32, s(16 + dst * 2), s15 + *(p++) = 0x92200f10u | (dst << 1); + + // s_add_u32 s33, s33, s32 + *(p++) = 0x80212021u; + + // s_mul_i32 s32, s(17 + dst * 2), s14 + *(p++) = 0x92200e11u | (dst << 1); + + // s_add_u32 s(17 + dst * 2), s33, s32 + *(p++) = 0x80112021u | (dst << 17); + + // s_mul_i32 s(16 + dst * 2), s(16 + dst * 2), s14 + *(p++) = 0x92100e10u | (dst << 1) | (dst << 17); + } + + // (12*7/8 + 8*1/8 + 28) + 24 = 63.5 bytes on average + return p; + } + opcode -= RANDOMX_FREQ_IMUL_M; + + if (opcode < RANDOMX_FREQ_IMULH_R) + { + *(p++) = 0xbe8e0110u | (dst << 1); // s_mov_b64 s[14:15], s[16 + dst * 2:17 + dst * 2] + *(p++) = 0xbea60110u | (src << 1); // s_mov_b64 s[38:39], s[16 + src * 2:17 + src * 2] + *(p++) = 0xbebc1e3au; // s_swappc_b64 s[60:61], s[58:59] + *(p++) = 0xbe90010eu | (dst << 17); // s_mov_b64 s[16 + dst * 2:17 + dst * 2], s[14:15] + + // 16 bytes + return p; + } + opcode -= RANDOMX_FREQ_IMULH_R; + + if (opcode < RANDOMX_FREQ_IMULH_M) + { + if (prefetch_vgpr_index >= 0) + { + if (src != dst) // p = 7/8 + p = jit_scratchpad_calc_address(p, src, inst.y, (mod % 4) ? ScratchpadL1Mask_reg : ScratchpadL2Mask_reg, batch_size); + else // p = 1/8 + p = jit_scratchpad_calc_fixed_address(p, inst.y & ScratchpadL3Mask, batch_size); + + p = jit_scratchpad_load(p, prefetch_vgpr_index ? prefetch_vgpr_index : 28); + } + + if (prefetch_vgpr_index <= 0) + { + p = jit_scratchpad_load2(p, prefetch_vgpr_index ? -prefetch_vgpr_index : 28, prefetch_vgpr_index ? vmcnt : 0); + + *(p++) = 0xbea60110u | (dst << 1); // s_mov_b64 s[38:39], s[16 + src * 2:17 + src * 2] + *(p++) = 0xbebc1e3au; // s_swappc_b64 s[60:61], s[58:59] + *(p++) = 0xbe90010eu | (dst << 17); // s_mov_b64 s[16 + dst * 2:17 + dst * 2], s[14:15] + } + + // (12*7/8 + 8*1/8 + 28) + 12 = 51.5 bytes on average + return p; + } + opcode -= RANDOMX_FREQ_IMULH_M; + + if (opcode < RANDOMX_FREQ_ISMULH_R) + { + *(p++) = 0xbe8e0110u | (dst << 1); // s_mov_b64 s[14:15], s[16 + dst * 2:17 + dst * 2] + *(p++) = 0xbea60110u | (src << 1); // s_mov_b64 s[38:39], s[16 + src * 2:17 + src * 2] + *(p++) = 0xbebc1e38u; // s_swappc_b64 s[60:61], s[56:57] + *(p++) = 0xbe90010eu | (dst << 17); // s_mov_b64 s[16 + dst * 2:17 + dst * 2], s[14:15] + + // 16 bytes + return p; + } + opcode -= RANDOMX_FREQ_ISMULH_R; + + if (opcode < RANDOMX_FREQ_ISMULH_M) + { + if (prefetch_vgpr_index >= 0) + { + if (src != dst) // p = 7/8 + p = jit_scratchpad_calc_address(p, src, inst.y, (mod % 4) ? ScratchpadL1Mask_reg : ScratchpadL2Mask_reg, batch_size); + else // p = 1/8 + p = jit_scratchpad_calc_fixed_address(p, inst.y & ScratchpadL3Mask, batch_size); + + p = jit_scratchpad_load(p, prefetch_vgpr_index ? prefetch_vgpr_index : 28); + } + + if (prefetch_vgpr_index <= 0) + { + p = jit_scratchpad_load2(p, prefetch_vgpr_index ? -prefetch_vgpr_index : 28, prefetch_vgpr_index ? vmcnt : 0); + + *(p++) = 0xbea60110u | (dst << 1); // s_mov_b64 s[38:39], s[16 + dst * 2:17 + dst * 2] + *(p++) = 0xbebc1e38u; // s_swappc_b64 s[60:61], s[56:57] + *(p++) = 0xbe90010eu | (dst << 17); // s_mov_b64 s[16 + dst * 2:17 + dst * 2], s[14:15] + } + + // (12*7/8 + 8*1/8 + 28) + 12 = 51.5 bytes on average + return p; + } + opcode -= RANDOMX_FREQ_ISMULH_M; + + if (opcode < RANDOMX_FREQ_IMUL_RCP) + { + if (inst.y & (inst.y - 1)) + { + const uint2 rcp_value = as_uint2(imul_rcp_value(inst.y)); + + *(p++) = 0xbea000ffu; // s_mov_b32 s32, imm32 + *(p++) = rcp_value.x; +#if GCN_VERSION >= 14 + *(p++) = 0x960f2010u | (dst << 1); // s_mul_hi_u32 s15, s(16 + dst * 2), s32 +#else + // v_mov_b32 v28, s32 + *(p++) = 0x7e380220u; + // v_mul_hi_u32 v28, v28, s(16 + dst * 2) + *(p++) = 0xd286001cu; + *(p++) = 0x0000211cu + (dst << 10); + // v_readlane_b32 s15, v28, 0 + *(p++) = 0xd289000fu; + *(p++) = 0x0001011cu; +#endif + *(p++) = 0x920eff10u | (dst << 1); // s_mul_i32 s14, s(16 + dst * 2), imm32 + *(p++) = rcp_value.y; + *(p++) = 0x800f0e0fu; // s_add_u32 s15, s15, s14 + *(p++) = 0x920e2011u | (dst << 1); // s_mul_i32 s14, s(17 + dst * 2), s32 + *(p++) = 0x80110e0fu | (dst << 17); // s_add_u32 s(17 + dst * 2), s15, s14 + *(p++) = 0x92102010u | (dst << 1) | (dst << 17);// s_mul_i32 s(16 + dst * 2), s(16 + dst * 2), s32 + } + + // 36 bytes + return p; + } + opcode -= RANDOMX_FREQ_IMUL_RCP; + + if (opcode < RANDOMX_FREQ_INEG_R) + { + *(p++) = 0x80901080u | (dst << 9) | (dst << 17); // s_sub_u32 s(16 + dst * 2), 0, s(16 + dst * 2) + *(p++) = 0x82911180u | (dst << 9) | (dst << 17); // s_subb_u32 s(17 + dst * 2), 0, s(17 + dst * 2) + + // 8 bytes + return p; + } + opcode -= RANDOMX_FREQ_INEG_R; + + if (opcode < RANDOMX_FREQ_IXOR_R) + { + if (src != dst) // p = 7/8 + { + // s_xor_b64 s[16 + dst * 2:17 + dst * 2], s[16 + dst * 2:17 + dst * 2], s[16 + src * 2:17 + src * 2] + *(p++) = 0x88901010u | (dst << 1) | (dst << 17) | (src << 9); + } + else // p = 1/8 + { + if (as_int(inst.y) < 0) // p = 1/2 + { + // s_mov_b32 s62, imm32 + *(p++) = 0xbebe00ffu; + *(p++) = inst.y; + + // s_xor_b64 s[16 + dst * 2:17 + dst * 2], s[16 + dst * 2:17 + dst * 2], s[62:63] + *(p++) = 0x88903e10u | (dst << 1) | (dst << 17); + } + else + { + // s_xor_b32 s(16 + dst * 2), s(16 + dst * 2), imm32 + *(p++) = 0x8810ff10u | (dst << 1) | (dst << 17); + *(p++) = inst.y; + } + } + + // 4*7/8 + 12/16 + 8/16 = 4.75 bytes on average + return p; + } + opcode -= RANDOMX_FREQ_IXOR_R; + + if (opcode < RANDOMX_FREQ_IXOR_M) + { + if (prefetch_vgpr_index >= 0) + { + if (src != dst) // p = 7/8 + p = jit_scratchpad_calc_address(p, src, inst.y, (mod % 4) ? ScratchpadL1Mask_reg : ScratchpadL2Mask_reg, batch_size); + else // p = 1/8 + p = jit_scratchpad_calc_fixed_address(p, inst.y & ScratchpadL3Mask, batch_size); + + p = jit_scratchpad_load(p, prefetch_vgpr_index ? prefetch_vgpr_index : 28); + } + + if (prefetch_vgpr_index <= 0) + { + p = jit_scratchpad_load2(p, prefetch_vgpr_index ? -prefetch_vgpr_index : 28, prefetch_vgpr_index ? vmcnt : 0); + + // s_xor_b64 s[16 + dst * 2:17 + dst * 2], s[16 + dst * 2:17 + dst * 2], s[14:15] + *(p++) = 0x88900e10u | (dst << 1) | (dst << 17); + } + + // (12*7/8 + 8*1/8 + 28) + 4 = 43.5 bytes on average + return p; + } + opcode -= RANDOMX_FREQ_IXOR_M; + + if (opcode < RANDOMX_FREQ_IROR_R + RANDOMX_FREQ_IROL_R) + { + if (src != dst) // p = 7/8 + { + if (opcode < RANDOMX_FREQ_IROR_R) + { + // s_lshr_b64 s[32:33], s[16 + dst * 2:17 + dst * 2], s(16 + src * 2) + *(p++) = 0x8fa01010u | (dst << 1) | (src << 9); + + // s_sub_u32 s15, 64, s(16 + src * 2) + *(p++) = 0x808f10c0u | (src << 9); + + // s_lshl_b64 s[34:35], s[16 + dst * 2:17 + dst * 2], s15 + *(p++) = 0x8ea20f10u | (dst << 1); + } + else + { + // s_lshl_b64 s[32:33], s[16 + dst * 2:17 + dst * 2], s(16 + src * 2) + *(p++) = 0x8ea01010u | (dst << 1) | (src << 9); + + // s_sub_u32 s15, 64, s(16 + src * 2) + *(p++) = 0x808f10c0u | (src << 9); + + // s_lshr_b64 s[34:35], s[16 + dst * 2:17 + dst * 2], s15 + *(p++) = 0x8fa20f10u | (dst << 1); + } + } + else // p = 1/8 + { + const uint shift = ((opcode < RANDOMX_FREQ_IROR_R) ? inst.y : -inst.y) & 63; + + // s_lshr_b64 s[32:33], s[16 + dst * 2:17 + dst * 2], shift + *(p++) = 0x8fa08010u | (dst << 1) | (shift << 8); + + // s_lshl_b64 s[34:35], s[16 + dst * 2:17 + dst * 2], 64 - shift + *(p++) = 0x8ea28010u | (dst << 1) | ((64 - shift) << 8); + } + + // s_or_b64 s[16 + dst * 2:17 + dst * 2], s[32:33], s[34:35] + *(p++) = 0x87902220u | (dst << 17); + + // 12*7/8 + 8/8 + 4 = 15.5 bytes on average + return p; + } + opcode -= RANDOMX_FREQ_IROR_R + RANDOMX_FREQ_IROL_R; + + if (opcode < RANDOMX_FREQ_ISWAP_R) + { + if (src != dst) + { + *(p++) = 0xbea00110u | (dst << 1); // s_mov_b64 s[32:33], s[16 + dst * 2:17 + dst * 2] + *(p++) = 0xbe900110u | (src << 1) | (dst << 17);// s_mov_b64 s[16 + dst * 2:17 + dst * 2], s[16 + src * 2:17 + src * 2] + *(p++) = 0xbe900120u | (src << 17); // s_mov_b64 s[16 + src * 2:17 + Src * 2], s[32:33] + } + + // 12*7/8 = 10.5 bytes on average + return p; + } + opcode -= RANDOMX_FREQ_ISWAP_R; + + if (opcode < RANDOMX_FREQ_FSWAP_R) + { + // ds_swizzle_b32 v(60 + dst * 2), v(60 + dst * 2) offset:0x8001 + *(p++) = 0xd87a8001u; + *(p++) = 0x3c00003cu + (dst << 1) + (dst << 25); + + // ds_swizzle_b32 v(61 + dst * 2), v(61 + dst * 2) offset:0x8001 + *(p++) = 0xd87a8001u; + *(p++) = 0x3d00003du + (dst << 1) + (dst << 25); + + // s_waitcnt lgkmcnt(0) + *(p++) = 0xbf8cc07fu; + + // 20 bytes + return p; + } + opcode -= RANDOMX_FREQ_FSWAP_R; + + if (opcode < RANDOMX_FREQ_FADD_R) + { + // v_add_f64 v[60 + dst * 2:61 + dst * 2], v[60 + dst * 2:61 + dst * 2], v[52 + src * 2:53 + src * 2] + *(p++) = 0xd280003cu + ((dst & 3) << 1); + *(p++) = 0x0002693cu + ((dst & 3) << 1) + ((src & 3) << 10); + + // 8 bytes + return p; + } + opcode -= RANDOMX_FREQ_FADD_R; + +)===" +R"===( + + if (opcode < RANDOMX_FREQ_FADD_M) + { + if (prefetch_vgpr_index >= 0) + { + p = jit_scratchpad_calc_address_fp(p, src, inst.y, (mod % 4) ? ScratchpadL1Mask_reg : ScratchpadL2Mask_reg, batch_size); + p = jit_scratchpad_load_fp(p, prefetch_vgpr_index ? prefetch_vgpr_index : 28); + } + + if (prefetch_vgpr_index <= 0) + { + p = jit_scratchpad_load2_fp(p, prefetch_vgpr_index ? -prefetch_vgpr_index : 28, prefetch_vgpr_index ? vmcnt : 0); + + // v_add_f64 v[60 + dst * 2:61 + dst * 2], v[60 + dst * 2:61 + dst * 2], v[28:29] + *(p++) = 0xd280003cu + ((dst & 3) << 1); + *(p++) = 0x0002393cu + ((dst & 3) << 1); + } + + // 32 + 8 = 40 bytes + return p; + } + opcode -= RANDOMX_FREQ_FADD_M; + + if (opcode < RANDOMX_FREQ_FSUB_R) + { + // v_add_f64 v[60 + dst * 2:61 + dst * 2], v[60 + dst * 2:61 + dst * 2], -v[52 + src * 2:53 + src * 2] + *(p++) = 0xd280003cu + ((dst & 3) << 1); + *(p++) = 0x4002693cu + ((dst & 3) << 1) + ((src & 3) << 10); + + // 8 bytes + return p; + } + opcode -= RANDOMX_FREQ_FSUB_R; + + if (opcode < RANDOMX_FREQ_FSUB_M) + { + if (prefetch_vgpr_index >= 0) + { + p = jit_scratchpad_calc_address_fp(p, src, inst.y, (mod % 4) ? ScratchpadL1Mask_reg : ScratchpadL2Mask_reg, batch_size); + p = jit_scratchpad_load_fp(p, prefetch_vgpr_index ? prefetch_vgpr_index : 28); + } + + if (prefetch_vgpr_index <= 0) + { + p = jit_scratchpad_load2_fp(p, prefetch_vgpr_index ? -prefetch_vgpr_index : 28, prefetch_vgpr_index ? vmcnt : 0); + + // v_add_f64 v[60 + dst * 2:61 + dst * 2], v[60 + dst * 2:61 + dst * 2], -v[28:29] + *(p++) = 0xd280003cu + ((dst & 3) << 1); + *(p++) = 0x4002393cu + ((dst & 3) << 1); + } + + // 32 + 8 = 40 bytes + return p; + } + opcode -= RANDOMX_FREQ_FSUB_M; + + if (opcode < RANDOMX_FREQ_FSCAL_R) + { + // v_xor_b32 v(61 + dst * 2), v(61 + dst * 2), v51 + *(p++) = 0x2a7a673du + ((dst & 3) << 1) + ((dst & 3) << 18); + + // 4 bytes + return p; + } + opcode -= RANDOMX_FREQ_FSCAL_R; + + if (opcode < RANDOMX_FREQ_FMUL_R) + { + // v_mul_f64 v[68 + dst * 2:69 + dst * 2], v[68 + dst * 2:69 + dst * 2], v[52 + src * 2:53 + src * 2] + *(p++) = 0xd2810044u + ((dst & 3) << 1); + *(p++) = 0x00026944u + ((dst & 3) << 1) + ((src & 3) << 10); + + // 8 bytes + return p; + } + opcode -= RANDOMX_FREQ_FMUL_R; + + if (opcode < RANDOMX_FREQ_FDIV_M) + { + if (prefetch_vgpr_index >= 0) + { + p = jit_scratchpad_calc_address_fp(p, src, inst.y, (mod % 4) ? ScratchpadL1Mask_reg : ScratchpadL2Mask_reg, batch_size); + p = jit_scratchpad_load_fp(p, prefetch_vgpr_index ? prefetch_vgpr_index : 28); + } + + if (prefetch_vgpr_index <= 0) + { + p = jit_scratchpad_load2_fp(p, prefetch_vgpr_index ? -prefetch_vgpr_index : 28, prefetch_vgpr_index ? vmcnt : 0); + + // s_swappc_b64 s[60:61], s[48 + dst * 2:49 + dst * 2] + *(p++) = 0xbebc1e30u + ((dst & 3) << 1); + } + + // 32 + 4 = 36 bytes + return p; + } + opcode -= RANDOMX_FREQ_FDIV_M; + + if (opcode < RANDOMX_FREQ_FSQRT_R) + { + // s_swappc_b64 s[60:61], s[40 + dst * 2:41 + dst * 2] + *(p++) = 0xbebc1e28u + ((dst & 3) << 1); + + // 4 bytes + return p; + } + opcode -= RANDOMX_FREQ_FSQRT_R; + + if (opcode < RANDOMX_FREQ_CBRANCH) + { + const int shift = (mod >> 4) + RANDOMX_JUMP_OFFSET; + uint imm = inst.y | (1u << shift); + imm &= ~(1u << (shift - 1)); + + // s_add_u32 s(16 + dst * 2), s(16 + dst * 2), imm32 + *(p++) = 0x8010ff10 | (dst << 1) | (dst << 17); + *(p++) = imm; + + // s_addc_u32 s(17 + dst * 2), s(17 + dst * 2), ((imm < 0) ? -1 : 0) + *(p++) = 0x82110011u | (dst << 1) | (dst << 17) | (((as_int(imm) < 0) ? 0xc1 : 0x80) << 8); + + const uint conditionMaskReg = 70 + (mod >> 4); + + // s_and_b32 s14, s(16 + dst * 2), conditionMaskReg + *(p++) = 0x860e0010u | (dst << 1) | (conditionMaskReg << 8); + + // s_cbranch_scc0 target + const int delta = ((last_branch_target - p) - 1); + *(p++) = 0xbf840000u | (delta & 0xFFFF); + + // 20 bytes + return p; + } + opcode -= RANDOMX_FREQ_CBRANCH; + + if (opcode < RANDOMX_FREQ_CFROUND) + { + const uint shift = inst.y & 63; + if (shift == 63) + { + *(p++) = 0x8e0e8110u | (src << 1); // s_lshl_b32 s14, s(16 + src * 2), 1 + *(p++) = 0x8f0f9f11u | (src << 1); // s_lshr_b32 s15, s(17 + src * 2), 31 + *(p++) = 0x870e0f0eu; // s_or_b32 s14, s14, s15 + *(p++) = 0x860e830eu; // s_and_b32 s14, s14, 3 + } + else + { + // s_bfe_u64 s[14:15], s[16:17], (shift,width=2) + *(p++) = 0x938eff10u | (src << 1); + *(p++) = shift | (2 << 16); + } + + // s_brev_b32 s14, s14 + *(p++) = 0xbe8e080eu; + + // s_lshr_b32 s66, s14, 30 + *(p++) = 0x8f429e0eu; + + // s_setreg_b32 hwreg(mode, 2, 2), s66 + *(p++) = 0xb9420881u; + + // 20 bytes + return p; + } + opcode -= RANDOMX_FREQ_CFROUND; + + if (opcode < RANDOMX_FREQ_ISTORE) + { + const uint mask = ((mod >> 4) < 14) ? ((mod % 4) ? ScratchpadL1Mask_reg : ScratchpadL2Mask_reg) : ScratchpadL3Mask_reg; + p = jit_scratchpad_calc_address(p, dst, inst.y, mask, batch_size); + + const uint vgpr_id = 48; + *(p++) = 0x7e000210u | (src << 1) | (vgpr_id << 17); // v_mov_b32 vgpr_id, s(16 + src * 2) + *(p++) = 0x7e020211u | (src << 1) | (vgpr_id << 17); // v_mov_b32 vgpr_id + 1, s(17 + src * 2) + + // v28 = offset + +#if GCN_VERSION >= 14 + // global_store_dwordx2 v28, v[vgpr_id:vgpr_id + 1], s[0:1] + *(p++) = 0xdc748000u; + *(p++) = 0x0000001cu | (vgpr_id << 8); +#else + // v_add_u32 v28, vcc, v28, v2 + *(p++) = 0x3238051cu; + // v_addc_u32 v29, vcc, 0, v3, vcc + *(p++) = 0x383a0680u; + // flat_store_dwordx2 v[28:29], v[vgpr_id:vgpr_id + 1] + *(p++) = 0xdc740000u; + *(p++) = 0x0000001cu | (vgpr_id << 8); +#endif + + // 28 bytes + return p; + } + opcode -= RANDOMX_FREQ_ISTORE; + + return p; +} + +int jit_prefetch_read( + __global uint2* p0, + const int prefetch_data_count, + const uint i, + const uint src, + const uint dst, + const uint2 inst, + const uint srcAvailableAt, + const uint scratchpadAvailableAt, + const uint scratchpadHighAvailableAt, + const int lastBranchTarget, + const int lastBranch) +{ + uint2 t; + t.x = (src == dst) ? (((inst.y & ScratchpadL3Mask) >= RANDOMX_SCRATCHPAD_L2) ? scratchpadHighAvailableAt : scratchpadAvailableAt) : max(scratchpadAvailableAt, srcAvailableAt); + t.y = i; + + const int t1 = t.x; + + if ((lastBranchTarget <= t1) && (t1 <= lastBranch)) + { + // Don't move prefetch inside previous branch scope + t.x = lastBranch + 1; + } + else if ((lastBranchTarget > lastBranch) && (t1 < lastBranchTarget)) + { + // Don't move prefetch outside current branch scope + t.x = lastBranchTarget; + } + + p0[prefetch_data_count] = t; + return prefetch_data_count + 1; +} + +)===" +R"===( + +__global uint* generate_jit_code(__global uint2* e, __global uint2* p0, __global uint* p, uint batch_size) +{ + int prefetch_data_count; + + #pragma unroll(1) + for (int pass = 0; pass < 2; ++pass) + { +#if RANDOMX_PROGRAM_SIZE > 256 + int registerLastChanged[8] = { -1, -1, -1, -1, -1, -1, -1, -1 }; +#else + ulong registerLastChanged = 0; + uint registerWasChanged = 0; +#endif + + uint scratchpadAvailableAt = 0; + uint scratchpadHighAvailableAt = 0; + + int lastBranchTarget = -1; + int lastBranch = -1; + +#if RANDOMX_PROGRAM_SIZE > 256 + int registerLastChangedAtBranchTarget[8] = { -1, -1, -1, -1, -1, -1, -1, -1 }; +#else + ulong registerLastChangedAtBranchTarget = 0; + uint registerWasChangedAtBranchTarget = 0; +#endif + uint scratchpadAvailableAtBranchTarget = 0; + uint scratchpadHighAvailableAtBranchTarget = 0; + + prefetch_data_count = 0; + + #pragma unroll(1) + for (uint i = 0; i < RANDOMX_PROGRAM_SIZE; ++i) + { + // Clean flags + if (pass == 0) + e[i].x &= ~(0xf8u << 8); + + uint2 inst = e[i]; + uint opcode = inst.x & 0xFF; + const uint dst = (inst.x >> 8) & 7; + const uint src = (inst.x >> 16) & 7; + const uint mod = inst.x >> 24; + + if (pass == 1) + { + // Branch target + if (inst.x & (0x20 << 8)) + { + lastBranchTarget = i; +#if RANDOMX_PROGRAM_SIZE > 256 + #pragma unroll + for (int j = 0; j < 8; ++j) + registerLastChangedAtBranchTarget[j] = registerLastChanged[j]; +#else + registerLastChangedAtBranchTarget = registerLastChanged; + registerWasChangedAtBranchTarget = registerWasChanged; +#endif + scratchpadAvailableAtBranchTarget = scratchpadAvailableAt; + scratchpadHighAvailableAtBranchTarget = scratchpadHighAvailableAt; + } + + // Branch + if (inst.x & (0x40 << 8)) + lastBranch = i; + } + +#if RANDOMX_PROGRAM_SIZE > 256 + const uint srcAvailableAt = registerLastChanged[src] + 1; + const uint dstAvailableAt = registerLastChanged[dst] + 1; +#else + const uint srcAvailableAt = (registerWasChanged & (1u << src)) ? (((registerLastChanged >> (src * 8)) & 0xFF) + 1) : 0; + const uint dstAvailableAt = (registerWasChanged & (1u << dst)) ? (((registerLastChanged >> (dst * 8)) & 0xFF) + 1) : 0; +#endif + + if (opcode < RANDOMX_FREQ_IADD_RS) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + continue; + } + opcode -= RANDOMX_FREQ_IADD_RS; + + if (opcode < RANDOMX_FREQ_IADD_M) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + if (pass == 1) + prefetch_data_count = jit_prefetch_read(p0, prefetch_data_count, i, src, dst, inst, srcAvailableAt, scratchpadAvailableAt, scratchpadHighAvailableAt, lastBranchTarget, lastBranch); + continue; + } + opcode -= RANDOMX_FREQ_IADD_M; + + if (opcode < RANDOMX_FREQ_ISUB_R) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + continue; + } + opcode -= RANDOMX_FREQ_ISUB_R; + + if (opcode < RANDOMX_FREQ_ISUB_M) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + if (pass == 1) + prefetch_data_count = jit_prefetch_read(p0, prefetch_data_count, i, src, dst, inst, srcAvailableAt, scratchpadAvailableAt, scratchpadHighAvailableAt, lastBranchTarget, lastBranch); + continue; + } + opcode -= RANDOMX_FREQ_ISUB_M; + + if (opcode < RANDOMX_FREQ_IMUL_R) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + continue; + } + opcode -= RANDOMX_FREQ_IMUL_R; + + if (opcode < RANDOMX_FREQ_IMUL_M) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + if (pass == 1) + prefetch_data_count = jit_prefetch_read(p0, prefetch_data_count, i, src, dst, inst, srcAvailableAt, scratchpadAvailableAt, scratchpadHighAvailableAt, lastBranchTarget, lastBranch); + continue; + } + opcode -= RANDOMX_FREQ_IMUL_M; + + if (opcode < RANDOMX_FREQ_IMULH_R) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + continue; + } + opcode -= RANDOMX_FREQ_IMULH_R; + + if (opcode < RANDOMX_FREQ_IMULH_M) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + if (pass == 1) + prefetch_data_count = jit_prefetch_read(p0, prefetch_data_count, i, src, dst, inst, srcAvailableAt, scratchpadAvailableAt, scratchpadHighAvailableAt, lastBranchTarget, lastBranch); + continue; + } + opcode -= RANDOMX_FREQ_IMULH_M; + + if (opcode < RANDOMX_FREQ_ISMULH_R) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + continue; + } + opcode -= RANDOMX_FREQ_ISMULH_R; + + if (opcode < RANDOMX_FREQ_ISMULH_M) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + if (pass == 1) + prefetch_data_count = jit_prefetch_read(p0, prefetch_data_count, i, src, dst, inst, srcAvailableAt, scratchpadAvailableAt, scratchpadHighAvailableAt, lastBranchTarget, lastBranch); + continue; + } + opcode -= RANDOMX_FREQ_ISMULH_M; + + if (opcode < RANDOMX_FREQ_IMUL_RCP) + { + if (inst.y & (inst.y - 1)) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + } + continue; + } + opcode -= RANDOMX_FREQ_IMUL_RCP; + + if (opcode < RANDOMX_FREQ_INEG_R + RANDOMX_FREQ_IXOR_R) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + continue; + } + opcode -= RANDOMX_FREQ_INEG_R + RANDOMX_FREQ_IXOR_R; + + if (opcode < RANDOMX_FREQ_IXOR_M) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + if (pass == 1) + prefetch_data_count = jit_prefetch_read(p0, prefetch_data_count, i, src, dst, inst, srcAvailableAt, scratchpadAvailableAt, scratchpadHighAvailableAt, lastBranchTarget, lastBranch); + continue; + } + opcode -= RANDOMX_FREQ_IXOR_M; + + if (opcode < RANDOMX_FREQ_IROR_R + RANDOMX_FREQ_IROL_R) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + continue; + } + opcode -= RANDOMX_FREQ_IROR_R + RANDOMX_FREQ_IROL_R; + + if (opcode < RANDOMX_FREQ_ISWAP_R) + { + if (src != dst) + { +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; + registerLastChanged[src] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerLastChanged = (registerLastChanged & ~(0xFFul << (src * 8))) | ((ulong)(i) << (src * 8)); + registerWasChanged |= (1u << dst) | (1u << src); +#endif + } + continue; + } + opcode -= RANDOMX_FREQ_ISWAP_R; + + if (opcode < RANDOMX_FREQ_FSWAP_R + RANDOMX_FREQ_FADD_R) + { + continue; + } + opcode -= RANDOMX_FREQ_FSWAP_R + RANDOMX_FREQ_FADD_R; + + if (opcode < RANDOMX_FREQ_FADD_M) + { + if (pass == 1) + prefetch_data_count = jit_prefetch_read(p0, prefetch_data_count, i, src, 0xFF, inst, srcAvailableAt, scratchpadAvailableAt, scratchpadHighAvailableAt, lastBranchTarget, lastBranch); + continue; + } + opcode -= RANDOMX_FREQ_FADD_M; + + if (opcode < RANDOMX_FREQ_FSUB_R) + { + continue; + } + opcode -= RANDOMX_FREQ_FSUB_R; + + if (opcode < RANDOMX_FREQ_FSUB_M) + { + if (pass == 1) + prefetch_data_count = jit_prefetch_read(p0, prefetch_data_count, i, src, 0xFF, inst, srcAvailableAt, scratchpadAvailableAt, scratchpadHighAvailableAt, lastBranchTarget, lastBranch); + continue; + } + opcode -= RANDOMX_FREQ_FSUB_M; + + if (opcode < RANDOMX_FREQ_FSCAL_R + RANDOMX_FREQ_FMUL_R) + { + continue; + } + opcode -= RANDOMX_FREQ_FSCAL_R + RANDOMX_FREQ_FMUL_R; + + if (opcode < RANDOMX_FREQ_FDIV_M) + { + if (pass == 1) + prefetch_data_count = jit_prefetch_read(p0, prefetch_data_count, i, src, 0xFF, inst, srcAvailableAt, scratchpadAvailableAt, scratchpadHighAvailableAt, lastBranchTarget, lastBranch); + continue; + } + opcode -= RANDOMX_FREQ_FDIV_M; + + if (opcode < RANDOMX_FREQ_FSQRT_R) + { + continue; + } + opcode -= RANDOMX_FREQ_FSQRT_R; + + if (opcode < RANDOMX_FREQ_CBRANCH) + { + if (pass == 0) + { + // Workaround for a bug in AMD 18.6.1 driver + volatile uint dstAvailableAt2 = dstAvailableAt; + + // Mark branch target + e[dstAvailableAt2].x |= (0x20 << 8); + + // Mark branch + e[i].x |= (0x40 << 8); + + // Set all registers as changed at this instruction as per RandomX specification +#if RANDOMX_PROGRAM_SIZE > 256 + #pragma unroll + for (int j = 0; j < 8; ++j) + registerLastChanged[j] = i; +#else + uint t = i | (i << 8); + t = t | (t << 16); + registerLastChanged = t; + registerLastChanged = registerLastChanged | (registerLastChanged << 32); + registerWasChanged = 0xFF; +#endif + } + else + { + // Update only registers which really changed inside this branch +#if RANDOMX_PROGRAM_SIZE > 256 + registerLastChanged[dst] = i; +#else + registerLastChanged = (registerLastChanged & ~(0xFFul << (dst * 8))) | ((ulong)(i) << (dst * 8)); + registerWasChanged |= 1u << dst; +#endif + + for (int reg = 0; reg < 8; ++reg) + { +#if RANDOMX_PROGRAM_SIZE > 256 + const uint availableAtBranchTarget = registerLastChangedAtBranchTarget[reg] + 1; + const uint availableAt = registerLastChanged[reg] + 1; + if (availableAt != availableAtBranchTarget) + { + registerLastChanged[reg] = i; + } +#else + const uint availableAtBranchTarget = (registerWasChangedAtBranchTarget & (1u << reg)) ? (((registerLastChangedAtBranchTarget >> (reg * 8)) & 0xFF) + 1) : 0; + const uint availableAt = (registerWasChanged & (1u << reg)) ? (((registerLastChanged >> (reg * 8)) & 0xFF) + 1) : 0; + if (availableAt != availableAtBranchTarget) + { + registerLastChanged = (registerLastChanged & ~(0xFFul << (reg * 8))) | ((ulong)(i) << (reg * 8)); + registerWasChanged |= 1u << reg; + } +#endif + } + + if (scratchpadAvailableAtBranchTarget != scratchpadAvailableAt) + scratchpadAvailableAt = i + 1; + + if (scratchpadHighAvailableAtBranchTarget != scratchpadHighAvailableAt) + scratchpadHighAvailableAt = i + 1; + } + continue; + } + opcode -= RANDOMX_FREQ_CBRANCH; + + if (opcode < RANDOMX_FREQ_CFROUND) + { + continue; + } + opcode -= RANDOMX_FREQ_CFROUND; + + if (opcode < RANDOMX_FREQ_ISTORE) + { + if (pass == 0) + { + // Mark ISTORE + e[i].x = inst.x | (0x80 << 8); + } + else + { + scratchpadAvailableAt = i + 1; + if ((mod >> 4) >= 14) + scratchpadHighAvailableAt = i + 1; + } + continue; + } + opcode -= RANDOMX_FREQ_ISTORE; + } + } + + // Sort p0 + uint prev = p0[0].x; + #pragma unroll(1) + for (int j = 1; j < prefetch_data_count; ++j) + { + uint2 cur = p0[j]; + if (cur.x >= prev) + { + prev = cur.x; + continue; + } + + int j1 = j - 1; + do { + p0[j1 + 1] = p0[j1]; + --j1; + } while ((j1 >= 0) && (p0[j1].x >= cur.x)); + p0[j1 + 1] = cur; + } + p0[prefetch_data_count].x = RANDOMX_PROGRAM_SIZE; + + __global int* prefecth_vgprs_stack = (__global int*)(p0 + prefetch_data_count + 1); + + // v86 - v127 will be used for global memory loads + enum { num_prefetch_vgprs = 21 }; + + #pragma unroll + for (int i = 0; i < num_prefetch_vgprs; ++i) + prefecth_vgprs_stack[i] = NUM_VGPR_REGISTERS - 2 - i * 2; + + __global int* prefetched_vgprs = prefecth_vgprs_stack + num_prefetch_vgprs; + + #pragma unroll(8) + for (int i = 0; i < RANDOMX_PROGRAM_SIZE; ++i) + prefetched_vgprs[i] = 0; + + int k = 0; + uint2 prefetch_data = p0[0]; + int mem_counter = 0; + int s_waitcnt_value = 63; + int num_prefetch_vgprs_available = num_prefetch_vgprs; + + __global uint* last_branch_target = p; + + const uint size_limit = (COMPILED_PROGRAM_SIZE - 200) / sizeof(uint); + __global uint* start_p = p; + + #pragma unroll(1) + for (int i = 0; i < RANDOMX_PROGRAM_SIZE; ++i) + { + const uint2 inst = e[i]; + + if (inst.x & (0x20 << 8)) + last_branch_target = p; + + bool done = false; + do { + uint2 jit_inst; + int jit_prefetch_vgpr_index; + int jit_vmcnt; + + if (!done && (prefetch_data.x == i) && (num_prefetch_vgprs_available > 0)) + { + ++mem_counter; + const int vgpr_id = prefecth_vgprs_stack[--num_prefetch_vgprs_available]; + prefetched_vgprs[prefetch_data.y] = vgpr_id | (mem_counter << 16); + + jit_inst = e[prefetch_data.y]; + jit_prefetch_vgpr_index = vgpr_id; + jit_vmcnt = mem_counter; + + s_waitcnt_value = 63; + + ++k; + prefetch_data = p0[k]; + } + else + { + const int prefetched_vgprs_data = prefetched_vgprs[i]; + const int vgpr_id = prefetched_vgprs_data & 0xFFFF; + const int prev_mem_counter = prefetched_vgprs_data >> 16; + if (vgpr_id) + prefecth_vgprs_stack[num_prefetch_vgprs_available++] = vgpr_id; + + if (inst.x & (0x80 << 8)) + { + ++mem_counter; + s_waitcnt_value = 63; + } + + const int vmcnt = mem_counter - prev_mem_counter; + + jit_inst = inst; + jit_prefetch_vgpr_index = -vgpr_id; + jit_vmcnt = (vmcnt < s_waitcnt_value) ? vmcnt : -1; + + if (vmcnt < s_waitcnt_value) + s_waitcnt_value = vmcnt; + + done = true; + } + + p = jit_emit_instruction(p, last_branch_target, jit_inst, jit_prefetch_vgpr_index, jit_vmcnt, batch_size); + if (p - start_p > size_limit) + { + // Code size limit exceeded!!! + // Jump back to randomx_run kernel + *(p++) = 0xbe801d0cu; // s_setpc_b64 s[12:13] + return p; + } + } while (!done); + } + + // Jump back to randomx_run kernel + *(p++) = 0xbe801d0cu; // s_setpc_b64 s[12:13] + return p; +} + +)===" +R"===( + +__attribute__((reqd_work_group_size(64, 1, 1))) +__kernel void randomx_jit(__global ulong* entropy, __global ulong* registers, __global uint2* intermediate_programs, __global uint* programs, uint batch_size, __global uint32_t* rounding, uint32_t iteration) +{ + const uint global_index = get_global_id(0) / 32; + const uint sub = get_global_id(0) % 32; + + if (sub != 0) + return; + + __global uint2* e = (__global uint2*)(entropy + global_index * (ENTROPY_SIZE / sizeof(ulong)) + (128 / sizeof(ulong))); + __global uint2* p0 = intermediate_programs + global_index * (INTERMEDIATE_PROGRAM_SIZE / sizeof(uint2)); + __global uint* p = programs + global_index * (COMPILED_PROGRAM_SIZE / sizeof(uint)); + + generate_jit_code(e, p0, p, batch_size); + + if (iteration == 0) + rounding[global_index] = 0; + + __global ulong* R = registers + global_index * 32; + entropy += global_index * (ENTROPY_SIZE / sizeof(ulong)); + + // Group R registers + R[0] = 0; + R[1] = 0; + R[2] = 0; + R[3] = 0; + R[4] = 0; + R[5] = 0; + R[6] = 0; + R[7] = 0; + + // Group A registers + __global double* A = (__global double*)(R + 24); + A[0] = getSmallPositiveFloatBits(entropy[0]); + A[1] = getSmallPositiveFloatBits(entropy[1]); + A[2] = getSmallPositiveFloatBits(entropy[2]); + A[3] = getSmallPositiveFloatBits(entropy[3]); + A[4] = getSmallPositiveFloatBits(entropy[4]); + A[5] = getSmallPositiveFloatBits(entropy[5]); + A[6] = getSmallPositiveFloatBits(entropy[6]); + A[7] = getSmallPositiveFloatBits(entropy[7]); + + // ma, mx + ((__global uint*)(R + 16))[0] = entropy[8] & CacheLineAlignMask; + ((__global uint*)(R + 16))[1] = entropy[10]; + + // address registers + uint addressRegisters = entropy[12]; + ((__global uint*)(R + 17))[0] = 0 + (addressRegisters & 1); + addressRegisters >>= 1; + ((__global uint*)(R + 17))[1] = 2 + (addressRegisters & 1); + addressRegisters >>= 1; + ((__global uint*)(R + 17))[2] = 4 + (addressRegisters & 1); + addressRegisters >>= 1; + ((__global uint*)(R + 17))[3] = 6 + (addressRegisters & 1); + + // dataset offset + ((__global uint*)(R + 19))[0] = (entropy[13] & DatasetExtraItems) * CacheLineSize; + + // eMask + R[20] = getFloatMask(entropy[14]); + R[21] = getFloatMask(entropy[15]); +} + +)===" diff --git a/src/backend/opencl/cl/rx/randomx_run_gfx803.asm b/src/backend/opencl/cl/rx/randomx_run_gfx803.asm new file mode 100644 index 00000000..47b41b57 --- /dev/null +++ b/src/backend/opencl/cl/rx/randomx_run_gfx803.asm @@ -0,0 +1,712 @@ +/* +Copyright (c) 2019 SChernykh + +This file is part of RandomX OpenCL. + +RandomX OpenCL is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +RandomX OpenCL is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with RandomX OpenCL. If not, see . +*/ + +.amdcl2 +.gpu GFX803 +.64bit +.arch_minor 0 +.arch_stepping 0 +.driver_version 203603 +.kernel randomx_run + .config + .dims x + .cws 64, 1, 1 + .sgprsnum 96 + # 6 waves per SIMD: 37-40 VGPRs + # 5 waves per SIMD: 41-48 VGPRs + # 4 waves per SIMD: 49-64 VGPRs + # 3 waves per SIMD: 65-84 VGPRs + # 2 waves per SIMD: 85-128 VGPRs + # 1 wave per SIMD: 129-256 VGPRs + .vgprsnum 128 + .localsize 256 + .floatmode 0xc0 + .pgmrsrc1 0x00ac035f + .pgmrsrc2 0x0000008c + .dx10clamp + .ieeemode + .useargs + .priority 0 + .arg _.global_offset_0, "size_t", long + .arg _.global_offset_1, "size_t", long + .arg _.global_offset_2, "size_t", long + .arg _.printf_buffer, "size_t", void*, global, , rdonly + .arg _.vqueue_pointer, "size_t", long + .arg _.aqlwrap_pointer, "size_t", long + .arg dataset, "uchar*", uchar*, global, const, rdonly + .arg scratchpad, "uchar*", uchar*, global, + .arg registers, "ulong*", ulong*, global, + .arg rounding_modes, "uint*", uint*, global, + .arg programs, "uint*", uint*, global, + .arg batch_size, "uint", uint + .arg rx_parameters, "uint", uint + .text + s_mov_b32 m0, 0x10000 + s_dcache_wb + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_icache_inv + s_branch begin + + # pgmrsrc2 = 0x00000090, bits 1:5 = 8, so first 8 SGPRs (s0-s7) contain user data + # s8 contains group id + # v0 contains local id +begin: + s_mov_b32 s8, s6 + v_lshlrev_b32 v1, 6, s8 + v_add_u32 v1, vcc, v1, v0 + s_load_dwordx2 s[0:1], s[4:5], 0x0 + s_load_dwordx2 s[2:3], s[4:5], 0x40 + s_load_dwordx2 s[64:65], s[4:5], 0x48 + s_waitcnt lgkmcnt(0) + + # load rounding mode + s_lshl_b32 s16, s8, 2 + s_add_u32 s64, s64, s16 + s_addc_u32 s65, s65, 0 + v_mov_b32 v8, s64 + v_mov_b32 v9, s65 + flat_load_dword v8, v[8:9] + s_waitcnt vmcnt(0) + v_readlane_b32 s66, v8, 0 + s_setreg_b32 hwreg(mode, 2, 2), s66 + s_mov_b32 s67, 0 + + # used in FSQRT_R to check for "positive normal value" (v_cmpx_class_f64) + s_mov_b32 s68, 256 + s_mov_b32 s69, 0 + + v_add_u32 v1, vcc, s0, v1 + v_lshrrev_b32 v2, 6, v1 + v_lshlrev_b32 v3, 5, v2 + v_and_b32 v1, 63, v1 + v_mov_b32 v4, 0 + v_lshlrev_b64 v[3:4], 3, v[3:4] + v_lshlrev_b32 v5, 4, v1 + v_add_u32 v3, vcc, s2, v3 + v_mov_b32 v6, s3 + v_addc_u32 v4, vcc, v6, v4, vcc + v_lshlrev_b32 v41, 2, v1 + v_add_u32 v6, vcc, v3, v41 + v_addc_u32 v7, vcc, v4, 0, vcc + flat_load_dword v6, v[6:7] + v_mov_b32 v0, 0 + s_waitcnt vmcnt(0) + ds_write_b32 v41, v6 + s_waitcnt lgkmcnt(0) + s_mov_b64 s[0:1], exec + v_cmpx_le_u32 s[2:3], v1, 7 + s_cbranch_execz program_end + + # rx_parameters + s_load_dword s20, s[4:5], 0x5c + s_waitcnt lgkmcnt(0) + + # Scratchpad L1 size + s_bfe_u32 s21, s20, 0x050000 + s_lshl_b32 s21, 1, s21 + + # Scratchpad L2 size + s_bfe_u32 s22, s20, 0x050005 + s_lshl_b32 s22, 1, s22 + + # Scratchpad L3 size + s_bfe_u32 s23, s20, 0x05000A + s_lshl_b32 s23, 1, s23 + + # program iterations + s_bfe_u32 s24, s20, 0x04000F + s_lshl_b32 s24, 1, s24 + + # Base address for scratchpads + s_add_u32 s2, s23, 64 + v_mul_hi_u32 v20, v2, s2 + v_mul_lo_u32 v2, v2, s2 + + # v41, v44 = 0 + v_mov_b32 v41, 0 + v_mov_b32 v44, 0 + + ds_read_b32 v6, v0 offset:152 + v_cmp_lt_u32 s[2:3], v1, 4 + ds_read2_b64 v[34:37], v0 offset0:18 offset1:16 + ds_read_b64 v[11:12], v0 offset:136 + s_movk_i32 s9, 0x0 + s_mov_b64 s[6:7], exec + s_andn2_b64 exec, s[6:7], s[2:3] + ds_read_b64 v[13:14], v0 offset:160 + s_andn2_b64 exec, s[6:7], exec + v_mov_b32 v13, 0 + v_mov_b32 v14, 0 + s_mov_b64 exec, s[6:7] + + # compiled program size + s_mov_b64 s[6:7], s[8:9] + s_mulk_i32 s6, 10048 + + v_add_u32 v5, vcc, v0, v5 + v_add_u32 v5, vcc, v5, 64 + s_mov_b64 s[8:9], exec + s_andn2_b64 exec, s[8:9], s[2:3] + ds_read_b64 v[15:16], v0 offset:168 + s_andn2_b64 exec, s[8:9], exec + v_mov_b32 v15, 0 + v_mov_b32 v16, 0 + s_mov_b64 exec, s[8:9] + s_load_dwordx4 s[8:11], s[4:5], 0x30 + + # batch_size + s_load_dword s16, s[4:5], 0x58 + + s_load_dwordx2 s[4:5], s[4:5], 0x50 + v_lshlrev_b32 v1, 3, v1 + v_add_u32 v17, vcc, v0, v1 + s_waitcnt lgkmcnt(0) + v_add_u32 v2, vcc, s10, v2 + v_mov_b32 v18, s11 + v_addc_u32 v18, vcc, v18, v20, vcc + v_mov_b32 v19, 0xffffff + v_add_u32 v6, vcc, s8, v6 + v_mov_b32 v20, s9 + v_addc_u32 v20, vcc, v20, 0, vcc + ds_read_b64 v[21:22], v17 + s_add_u32 s4, s4, s6 + s_addc_u32 s5, s5, s7 + v_cndmask_b32 v19, v19, -1, s[2:3] + v_lshlrev_b32 v8, 3, v35 + v_lshlrev_b32 v7, 3, v34 + v_lshlrev_b32 v12, 3, v12 + v_lshlrev_b32 v10, 3, v11 + v_add_u32 v8, vcc, v8, v0 + v_add_u32 v7, vcc, v7, v0 + v_add_u32 v12, vcc, v12, v0 + v_add_u32 v0, vcc, v10, v0 + v_mov_b32 v10, v36 + v_mov_b32 v23, v37 + + # loop counter + s_sub_u32 s2, s24, 1 + + # batch_size + s_mov_b32 s3, s16 + + # Scratchpad masks for scratchpads + v_sub_u32 v38, vcc, s21, 8 + v_sub_u32 v39, vcc, s22, 8 + v_sub_u32 v50, vcc, s23, 8 + + # mask for FSCAL_R + v_mov_b32 v51, 0x80F00000 + + # swap v3 and v18 + v_mov_b32 v52, v3 + v_mov_b32 v3, v18 + v_mov_b32 v18, v52 + + # load scratchpad base address + v_readlane_b32 s0, v2, 0 + v_readlane_b32 s1, v3, 0 + + # save current executiom mask + s_mov_b64 s[36:37], exec + + # v41 = 0 on lane 0, set it to 8 on lane 1 + # v44 = 0 on lane 0, set it to 4 on lane 1 + s_mov_b64 exec, 2 + v_mov_b32 v41, 8 + v_mov_b32 v44, 4 + + # load group A registers + # Read low 8 bytes into lane 0 and high 8 bytes into lane 1 + s_mov_b64 exec, 3 + ds_read2_b64 v[52:55], v41 offset0:24 offset1:26 + ds_read2_b64 v[56:59], v41 offset0:28 offset1:30 + + # xmantissaMask + v_mov_b32 v77, (1 << 24) - 1 + + # xexponentMask + ds_read_b64 v[78:79], v41 offset:160 + + # Restore execution mask + s_mov_b64 exec, s[36:37] + + # sign mask (used in FSQRT_R) + v_mov_b32 v82, 0x80000000 + + # High 32 bits of "1.0" constant (used in FDIV_M) + v_mov_b32 v83, (1023 << 20) + + # Used to multiply FP64 values by 0.5 + v_mov_b32 v84, (1 << 20) + + s_getpc_b64 s[14:15] +cur_addr: + + # get addresses of FSQRT_R subroutines + s_add_u32 s40, s14, fsqrt_r_sub0 - cur_addr + s_addc_u32 s41, s15, 0 + s_add_u32 s42, s14, fsqrt_r_sub1 - cur_addr + s_addc_u32 s43, s15, 0 + s_add_u32 s44, s14, fsqrt_r_sub2 - cur_addr + s_addc_u32 s45, s15, 0 + s_add_u32 s46, s14, fsqrt_r_sub3 - cur_addr + s_addc_u32 s47, s15, 0 + + # get addresses of FDIV_M subroutines + s_add_u32 s48, s14, fdiv_m_sub0 - cur_addr + s_addc_u32 s49, s15, 0 + s_add_u32 s50, s14, fdiv_m_sub1 - cur_addr + s_addc_u32 s51, s15, 0 + s_add_u32 s52, s14, fdiv_m_sub2 - cur_addr + s_addc_u32 s53, s15, 0 + s_add_u32 s54, s14, fdiv_m_sub3 - cur_addr + s_addc_u32 s55, s15, 0 + + # get address for ISMULH_R subroutine + s_add_u32 s56, s14, ismulh_r_sub - cur_addr + s_addc_u32 s57, s15, 0 + + # get address for IMULH_R subroutine + s_add_u32 s58, s14, imulh_r_sub - cur_addr + s_addc_u32 s59, s15, 0 + + # used in IXOR_R instruction + s_mov_b32 s63, -1 + + # used in CBRANCH instruction + s_mov_b32 s70, (0xFF << 8) + s_mov_b32 s71, (0xFF << 9) + s_mov_b32 s72, (0xFF << 10) + s_mov_b32 s73, (0xFF << 11) + s_mov_b32 s74, (0xFF << 12) + s_mov_b32 s75, (0xFF << 13) + s_mov_b32 s76, (0xFF << 14) + s_mov_b32 s77, (0xFF << 15) + s_mov_b32 s78, (0xFF << 16) + s_mov_b32 s79, (0xFF << 17) + s_mov_b32 s80, (0xFF << 18) + s_mov_b32 s81, (0xFF << 19) + s_mov_b32 s82, (0xFF << 20) + s_mov_b32 s83, (0xFF << 21) + s_mov_b32 s84, (0xFF << 22) + s_mov_b32 s85, (0xFF << 23) + + # ScratchpadL3Mask64 + s_sub_u32 s86, s23, 64 + +main_loop: + # const uint2 spMix = as_uint2(R[readReg0] ^ R[readReg1]); + ds_read_b64 v[24:25], v0 + ds_read_b64 v[26:27], v12 + s_waitcnt lgkmcnt(0) + v_xor_b32 v25, v27, v25 + v_xor_b32 v24, v26, v24 + + # spAddr1 ^= spMix.y; + # spAddr0 ^= spMix.x; + v_xor_b32 v10, v25, v10 + v_xor_b32 v23, v24, v23 + + # spAddr1 &= ScratchpadL3Mask64; + # spAddr0 &= ScratchpadL3Mask64; + v_and_b32 v10, s86, v10 + v_and_b32 v23, s86, v23 + + # Offset for scratchpads + # offset1 = spAddr1 + sub * 8 + # offset0 = spAddr0 + sub * 8 + v_add_u32 v10, vcc, v10, v1 + v_add_u32 v23, vcc, v23, v1 + + # __global ulong* p1 = (__global ulong*)(scratchpad + offset1); + # __global ulong* p0 = (__global ulong*)(scratchpad + offset0); + v_add_u32 v26, vcc, v2, v10 + v_addc_u32 v27, vcc, v3, 0, vcc + v_add_u32 v23, vcc, v2, v23 + v_addc_u32 v24, vcc, v3, 0, vcc + + # load from spAddr1 + flat_load_dwordx2 v[28:29], v[26:27] + + # load from spAddr0 + flat_load_dwordx2 v[30:31], v[23:24] + s_waitcnt vmcnt(1) + + v_cvt_f64_i32 v[32:33], v28 + v_cvt_f64_i32 v[28:29], v29 + s_waitcnt vmcnt(0) + + # R[sub] ^= *p0; + v_xor_b32 v34, v21, v30 + v_xor_b32 v35, v22, v31 + + v_add_u32 v22, vcc, v6, v36 + v_addc_u32 v25, vcc, v20, 0, vcc + v_add_u32 v21, vcc, v22, v1 + v_addc_u32 v22, vcc, v25, 0, vcc + flat_load_dwordx2 v[21:22], v[21:22] + v_or_b32 v30, v32, v13 + v_and_b32 v31, v33, v19 + v_or_b32 v31, v31, v14 + v_or_b32 v28, v28, v15 + v_and_b32 v29, v29, v19 + v_or_b32 v29, v29, v16 + ds_write2_b64 v5, v[30:31], v[28:29] offset1:1 + s_waitcnt lgkmcnt(0) + + # Program 0 + + # load group F,E registers + # Read low 8 bytes into lane 0 and high 8 bytes into lane 1 + s_mov_b64 exec, 3 + ds_read2_b64 v[60:63], v41 offset0:8 offset1:10 + ds_read2_b64 v[64:67], v41 offset0:12 offset1:14 + ds_read2_b64 v[68:71], v41 offset0:16 offset1:18 + ds_read2_b64 v[72:75], v41 offset0:20 offset1:22 + + # load VM integer registers + v_readlane_b32 s16, v34, 0 + v_readlane_b32 s17, v35, 0 + v_readlane_b32 s18, v34, 1 + v_readlane_b32 s19, v35, 1 + v_readlane_b32 s20, v34, 2 + v_readlane_b32 s21, v35, 2 + v_readlane_b32 s22, v34, 3 + v_readlane_b32 s23, v35, 3 + v_readlane_b32 s24, v34, 4 + v_readlane_b32 s25, v35, 4 + v_readlane_b32 s26, v34, 5 + v_readlane_b32 s27, v35, 5 + v_readlane_b32 s28, v34, 6 + v_readlane_b32 s29, v35, 6 + v_readlane_b32 s30, v34, 7 + v_readlane_b32 s31, v35, 7 + + s_waitcnt lgkmcnt(0) + + # call JIT code + s_swappc_b64 s[12:13], s[4:5] + + # Write out group F,E registers + # Write low 8 bytes from lane 0 and high 8 bytes from lane 1 + ds_write2_b64 v41, v[60:61], v[62:63] offset0:8 offset1:10 + ds_write2_b64 v41, v[64:65], v[66:67] offset0:12 offset1:14 + ds_write2_b64 v41, v[68:69], v[70:71] offset0:16 offset1:18 + ds_write2_b64 v41, v[72:73], v[74:75] offset0:20 offset1:22 + + # store VM integer registers + v_writelane_b32 v28, s16, 0 + v_writelane_b32 v29, s17, 0 + v_writelane_b32 v28, s18, 1 + v_writelane_b32 v29, s19, 1 + v_writelane_b32 v28, s20, 2 + v_writelane_b32 v29, s21, 2 + v_writelane_b32 v28, s22, 3 + v_writelane_b32 v29, s23, 3 + v_writelane_b32 v28, s24, 4 + v_writelane_b32 v29, s25, 4 + v_writelane_b32 v28, s26, 5 + v_writelane_b32 v29, s27, 5 + v_writelane_b32 v28, s28, 6 + v_writelane_b32 v29, s29, 6 + v_writelane_b32 v28, s30, 7 + v_writelane_b32 v29, s31, 7 + + # Restore execution mask + s_mov_b64 exec, s[36:37] + + # Write out VM integer registers + ds_write_b64 v17, v[28:29] + + s_waitcnt lgkmcnt(0) + v_xor_b32 v21, v28, v21 + v_xor_b32 v22, v29, v22 + ds_read_b32 v28, v7 + ds_read_b32 v29, v8 + ds_write_b64 v17, v[21:22] + s_waitcnt lgkmcnt(1) + ds_read2_b64 v[30:33], v17 offset0:8 offset1:16 + v_xor_b32 v10, v28, v37 + s_waitcnt lgkmcnt(0) + v_xor_b32 v30, v32, v30 + v_xor_b32 v31, v33, v31 + v_xor_b32 v10, v10, v29 + flat_store_dwordx2 v[26:27], v[21:22] + v_and_b32 v10, 0x7fffffc0, v10 + flat_store_dwordx2 v[23:24], v[30:31] + s_cmp_eq_u32 s2, 0 + s_cbranch_scc1 main_loop_end + s_sub_i32 s2, s2, 1 + v_mov_b32 v37, v36 + v_mov_b32 v23, 0 + v_mov_b32 v36, v10 + v_mov_b32 v10, 0 + s_branch main_loop +main_loop_end: + + v_add_u32 v0, vcc, v18, v1 + v_addc_u32 v1, vcc, v4, 0, vcc + flat_store_dwordx2 v[0:1], v[21:22] + v_add_u32 v0, vcc, v0, 64 + v_addc_u32 v1, vcc, v1, 0, vcc + flat_store_dwordx2 v[0:1], v[30:31] + v_add_u32 v0, vcc, v0, 64 + v_addc_u32 v1, vcc, v1, 0, vcc + flat_store_dwordx2 v[0:1], v[32:33] + + # store rounding mode + v_mov_b32 v0, s64 + v_mov_b32 v1, s65 + v_mov_b32 v2, s66 + flat_store_dword v[0:1], v2 + +program_end: + s_endpgm + +fsqrt_r_sub0: + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rsq_f64 v[28:29], v[68:69] + + # Improve initial approximation (can be skipped) + #v_mul_f64 v[42:43], v[28:29], v[68:69] + #v_mul_f64 v[48:49], v[28:29], -0.5 + #v_fma_f64 v[48:49], v[48:49], v[42:43], 0.5 + #v_fma_f64 v[28:29], v[28:29], v[48:49], v[28:29] + + v_mul_f64 v[42:43], v[28:29], v[68:69] + v_mov_b32 v48, v28 + v_sub_u32 v49, vcc, v29, v84 + v_mov_b32 v46, v28 + v_xor_b32 v47, v49, v82 + v_fma_f64 v[46:47], v[46:47], v[42:43], 0.5 + v_fma_f64 v[42:43], v[42:43], v[46:47], v[42:43] + v_fma_f64 v[48:49], v[48:49], v[46:47], v[48:49] + v_fma_f64 v[46:47], -v[42:43], v[42:43], v[68:69] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[46:47], v[48:49], v[42:43] + v_cmpx_class_f64 s[14:15], v[68:69], s[68:69] + v_mov_b32 v68, v42 + v_mov_b32 v69, v43 + s_mov_b64 exec, 3 + s_setpc_b64 s[60:61] + +fsqrt_r_sub1: + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rsq_f64 v[28:29], v[70:71] + + # Improve initial approximation (can be skipped) + #v_mul_f64 v[42:43], v[28:29], v[70:71] + #v_mul_f64 v[48:49], v[28:29], -0.5 + #v_fma_f64 v[48:49], v[48:49], v[42:43], 0.5 + #v_fma_f64 v[28:29], v[28:29], v[48:49], v[28:29] + + v_mul_f64 v[42:43], v[28:29], v[70:71] + v_mov_b32 v48, v28 + v_sub_u32 v49, vcc, v29, v84 + v_mov_b32 v46, v28 + v_xor_b32 v47, v49, v82 + v_fma_f64 v[46:47], v[46:47], v[42:43], 0.5 + v_fma_f64 v[42:43], v[42:43], v[46:47], v[42:43] + v_fma_f64 v[48:49], v[48:49], v[46:47], v[48:49] + v_fma_f64 v[46:47], -v[42:43], v[42:43], v[70:71] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[46:47], v[48:49], v[42:43] + v_cmpx_class_f64 s[14:15], v[70:71], s[68:69] + v_mov_b32 v70, v42 + v_mov_b32 v71, v43 + s_mov_b64 exec, 3 + s_setpc_b64 s[60:61] + +fsqrt_r_sub2: + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rsq_f64 v[28:29], v[72:73] + + # Improve initial approximation (can be skipped) + #v_mul_f64 v[42:43], v[28:29], v[72:73] + #v_mul_f64 v[48:49], v[28:29], -0.5 + #v_fma_f64 v[48:49], v[48:49], v[42:43], 0.5 + #v_fma_f64 v[28:29], v[28:29], v[48:49], v[28:29] + + v_mul_f64 v[42:43], v[28:29], v[72:73] + v_mov_b32 v48, v28 + v_sub_u32 v49, vcc, v29, v84 + v_mov_b32 v46, v28 + v_xor_b32 v47, v49, v82 + v_fma_f64 v[46:47], v[46:47], v[42:43], 0.5 + v_fma_f64 v[42:43], v[42:43], v[46:47], v[42:43] + v_fma_f64 v[48:49], v[48:49], v[46:47], v[48:49] + v_fma_f64 v[46:47], -v[42:43], v[42:43], v[72:73] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[46:47], v[48:49], v[42:43] + v_cmpx_class_f64 s[14:15], v[72:73], s[68:69] + v_mov_b32 v72, v42 + v_mov_b32 v73, v43 + s_mov_b64 exec, 3 + s_setpc_b64 s[60:61] + +fsqrt_r_sub3: + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rsq_f64 v[28:29], v[74:75] + + # Improve initial approximation (can be skipped) + #v_mul_f64 v[42:43], v[28:29], v[74:75] + #v_mul_f64 v[48:49], v[28:29], -0.5 + #v_fma_f64 v[48:49], v[48:49], v[42:43], 0.5 + #v_fma_f64 v[28:29], v[28:29], v[48:49], v[28:29] + + v_mul_f64 v[42:43], v[28:29], v[74:75] + v_mov_b32 v48, v28 + v_sub_u32 v49, vcc, v29, v84 + v_mov_b32 v46, v28 + v_xor_b32 v47, v49, v82 + v_fma_f64 v[46:47], v[46:47], v[42:43], 0.5 + v_fma_f64 v[42:43], v[42:43], v[46:47], v[42:43] + v_fma_f64 v[48:49], v[48:49], v[46:47], v[48:49] + v_fma_f64 v[46:47], -v[42:43], v[42:43], v[74:75] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[46:47], v[48:49], v[42:43] + v_cmpx_class_f64 s[14:15], v[74:75], s[68:69] + v_mov_b32 v74, v42 + v_mov_b32 v75, v43 + s_mov_b64 exec, 3 + s_setpc_b64 s[60:61] + +fdiv_m_sub0: + v_or_b32 v28, v28, v78 + v_and_b32 v29, v29, v77 + v_or_b32 v29, v29, v79 + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rcp_f64 v[48:49], v[28:29] + v_fma_f64 v[80:81], -v[28:29], v[48:49], 1.0 + v_fma_f64 v[48:49], v[48:49], v[80:81], v[48:49] + v_mul_f64 v[80:81], v[68:69], v[48:49] + v_fma_f64 v[42:43], -v[28:29], v[80:81], v[68:69] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[42:43], v[48:49], v[80:81] + v_div_fixup_f64 v[80:81], v[42:43], v[28:29], v[68:69] + v_cmpx_eq_f64 s[14:15], v[68:69], v[28:29] + v_mov_b32 v80, 0 + v_mov_b32 v81, v83 + s_mov_b64 exec, 3 + v_mov_b32 v68, v80 + v_mov_b32 v69, v81 + s_setpc_b64 s[60:61] + +fdiv_m_sub1: + v_or_b32 v28, v28, v78 + v_and_b32 v29, v29, v77 + v_or_b32 v29, v29, v79 + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rcp_f64 v[48:49], v[28:29] + v_fma_f64 v[80:81], -v[28:29], v[48:49], 1.0 + v_fma_f64 v[48:49], v[48:49], v[80:81], v[48:49] + v_mul_f64 v[80:81], v[70:71], v[48:49] + v_fma_f64 v[42:43], -v[28:29], v[80:81], v[70:71] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[42:43], v[48:49], v[80:81] + v_div_fixup_f64 v[80:81], v[42:43], v[28:29], v[70:71] + v_cmpx_eq_f64 s[14:15], v[70:71], v[28:29] + v_mov_b32 v80, 0 + v_mov_b32 v81, v83 + s_mov_b64 exec, 3 + v_mov_b32 v70, v80 + v_mov_b32 v71, v81 + s_setpc_b64 s[60:61] + +fdiv_m_sub2: + v_or_b32 v28, v28, v78 + v_and_b32 v29, v29, v77 + v_or_b32 v29, v29, v79 + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rcp_f64 v[48:49], v[28:29] + v_fma_f64 v[80:81], -v[28:29], v[48:49], 1.0 + v_fma_f64 v[48:49], v[48:49], v[80:81], v[48:49] + v_mul_f64 v[80:81], v[72:73], v[48:49] + v_fma_f64 v[42:43], -v[28:29], v[80:81], v[72:73] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[42:43], v[48:49], v[80:81] + v_div_fixup_f64 v[80:81], v[42:43], v[28:29], v[72:73] + v_cmpx_eq_f64 s[14:15], v[72:73], v[28:29] + v_mov_b32 v80, 0 + v_mov_b32 v81, v83 + s_mov_b64 exec, 3 + v_mov_b32 v72, v80 + v_mov_b32 v73, v81 + s_setpc_b64 s[60:61] + +fdiv_m_sub3: + v_or_b32 v28, v28, v78 + v_and_b32 v29, v29, v77 + v_or_b32 v29, v29, v79 + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rcp_f64 v[48:49], v[28:29] + v_fma_f64 v[80:81], -v[28:29], v[48:49], 1.0 + v_fma_f64 v[48:49], v[48:49], v[80:81], v[48:49] + v_mul_f64 v[80:81], v[74:75], v[48:49] + v_fma_f64 v[42:43], -v[28:29], v[80:81], v[74:75] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[42:43], v[48:49], v[80:81] + v_div_fixup_f64 v[80:81], v[42:43], v[28:29], v[74:75] + v_cmpx_eq_f64 s[14:15], v[74:75], v[28:29] + v_mov_b32 v80, 0 + v_mov_b32 v81, v83 + s_mov_b64 exec, 3 + v_mov_b32 v74, v80 + v_mov_b32 v75, v81 + s_setpc_b64 s[60:61] + +ismulh_r_sub: + s_mov_b64 exec, 1 + v_mov_b32 v45, s14 + v_mul_hi_u32 v40, s38, v45 + v_mov_b32 v47, s15 + v_mad_u64_u32 v[42:43], s[32:33], s38, v47, v[40:41] + v_mov_b32 v40, v42 + v_mad_u64_u32 v[45:46], s[32:33], s39, v45, v[40:41] + v_mad_u64_u32 v[42:43], s[32:33], s39, v47, v[43:44] + v_add_u32 v42, vcc, v42, v46 + v_addc_u32 v43, vcc, 0, v43, vcc + v_readlane_b32 s32, v42, 0 + v_readlane_b32 s33, v43, 0 + s_cmp_lt_i32 s15, 0 + s_cselect_b64 s[34:35], s[38:39], 0 + s_sub_u32 s32, s32, s34 + s_subb_u32 s33, s33, s35 + s_cmp_lt_i32 s39, 0 + s_cselect_b64 s[34:35], s[14:15], 0 + s_sub_u32 s14, s32, s34 + s_subb_u32 s15, s33, s35 + s_mov_b64 exec, 3 + s_setpc_b64 s[60:61] + +imulh_r_sub: + s_mov_b64 exec, 1 + v_mov_b32 v45, s38 + v_mul_hi_u32 v40, s14, v45 + v_mov_b32 v47, s39 + v_mad_u64_u32 v[42:43], s[32:33], s14, v47, v[40:41] + v_mov_b32 v40, v42 + v_mad_u64_u32 v[45:46], s[32:33], s15, v45, v[40:41] + v_mad_u64_u32 v[42:43], s[32:33], s15, v47, v[43:44] + v_add_u32 v42, vcc, v42, v46 + v_addc_u32 v43, vcc, 0, v43, vcc + v_readlane_b32 s14, v42, 0 + v_readlane_b32 s15, v43, 0 + s_mov_b64 exec, 3 + s_setpc_b64 s[60:61] diff --git a/src/backend/opencl/cl/rx/randomx_run_gfx803.h b/src/backend/opencl/cl/rx/randomx_run_gfx803.h new file mode 100644 index 00000000..0338efa9 --- /dev/null +++ b/src/backend/opencl/cl/rx/randomx_run_gfx803.h @@ -0,0 +1,218 @@ +/* +This file was auto-generated from randomx_run_gfx803.asm: + +clrxasm randomx_run_gfx803.asm -o randomx_run_gfx803.bin +bin2h -c randomx_run_gfx803_bin < randomx_run_gfx803.bin > randomx_run_gfx803.h + +clrxasm can be downloaded here: https://github.com/CLRX/CLRX-mirror/releases +bin2h can be downloaded here: http://www.deadnode.org/sw/bin2h/ +*/ + +static unsigned char randomx_run_gfx803_bin[]={ +0x7f,0x45,0x4c,0x46,0x02,0x01,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x00,0x5b,0xaf,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xe8,0x17,0x00,0x00,0x00,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x07,0x00,0x01,0x00,0x00 +,0x2e,0x73,0x68,0x73,0x74,0x72,0x74,0x61,0x62,0x00,0x2e,0x73,0x74,0x72,0x74,0x61,0x62,0x00,0x2e,0x73,0x79,0x6d,0x74,0x61,0x62,0x00,0x2e,0x63,0x6f,0x6d,0x6d,0x65 +,0x6e,0x74,0x00,0x2e,0x72,0x6f,0x64,0x61,0x74,0x61,0x00,0x2e,0x74,0x65,0x78,0x74,0x00,0x00,0x5f,0x5f,0x4f,0x70,0x65,0x6e,0x43,0x4c,0x5f,0x26,0x5f,0x5f,0x4f,0x70 +,0x65,0x6e,0x43,0x4c,0x5f,0x72,0x61,0x6e,0x64,0x6f,0x6d,0x78,0x5f,0x72,0x75,0x6e,0x5f,0x6b,0x65,0x72,0x6e,0x65,0x6c,0x5f,0x6d,0x65,0x74,0x61,0x64,0x61,0x74,0x61 +,0x00,0x61,0x63,0x6c,0x5f,0x76,0x65,0x72,0x73,0x69,0x6f,0x6e,0x5f,0x73,0x74,0x72,0x69,0x6e,0x67,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3b +,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x01,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41 +,0x4d,0x44,0x2d,0x43,0x4f,0x4d,0x50,0x2d,0x4c,0x49,0x42,0x2d,0x76,0x30,0x2e,0x38,0x20,0x28,0x30,0x2e,0x30,0x2e,0x53,0x43,0x5f,0x42,0x55,0x49,0x4c,0x44,0x5f,0x4e +,0x55,0x4d,0x42,0x45,0x52,0x29,0x10,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x68,0x00 +,0x00,0x00,0x24,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x40,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x5f,0x5f,0x4f,0x70,0x65,0x6e,0x43,0x4c,0x5f,0x64 +,0x75,0x6d,0x6d,0x79,0x5f,0x6b,0x65,0x72,0x6e,0x65,0x6c,0x00,0x67,0x65,0x6e,0x65,0x72,0x69,0x63,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00 +,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x10,0x00,0x00 +,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x0f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00 +,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x10,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00 +,0x00,0x01,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x50,0x00,0x00 +,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x01,0x00,0x00 +,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00 +,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x09,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00 +,0x00,0x01,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x90,0x00,0x00 +,0x00,0x07,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x04,0x00,0x00 +,0x00,0x08,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x0d,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00 +,0x00,0x01,0x00,0x00,0x00,0xc0,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x5f,0x2e,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x5f,0x6f,0x66,0x66,0x73,0x65,0x74,0x5f,0x30,0x00,0x73,0x69,0x7a,0x65,0x5f,0x74,0x00,0x5f,0x2e +,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x5f,0x6f,0x66,0x66,0x73,0x65,0x74,0x5f,0x31,0x00,0x73,0x69,0x7a,0x65,0x5f,0x74,0x00,0x5f,0x2e,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x5f +,0x6f,0x66,0x66,0x73,0x65,0x74,0x5f,0x32,0x00,0x73,0x69,0x7a,0x65,0x5f,0x74,0x00,0x5f,0x2e,0x70,0x72,0x69,0x6e,0x74,0x66,0x5f,0x62,0x75,0x66,0x66,0x65,0x72,0x00 +,0x73,0x69,0x7a,0x65,0x5f,0x74,0x00,0x5f,0x2e,0x76,0x71,0x75,0x65,0x75,0x65,0x5f,0x70,0x6f,0x69,0x6e,0x74,0x65,0x72,0x00,0x73,0x69,0x7a,0x65,0x5f,0x74,0x00,0x5f +,0x2e,0x61,0x71,0x6c,0x77,0x72,0x61,0x70,0x5f,0x70,0x6f,0x69,0x6e,0x74,0x65,0x72,0x00,0x73,0x69,0x7a,0x65,0x5f,0x74,0x00,0x64,0x61,0x74,0x61,0x73,0x65,0x74,0x00 +,0x75,0x63,0x68,0x61,0x72,0x2a,0x00,0x73,0x63,0x72,0x61,0x74,0x63,0x68,0x70,0x61,0x64,0x00,0x75,0x63,0x68,0x61,0x72,0x2a,0x00,0x72,0x65,0x67,0x69,0x73,0x74,0x65 +,0x72,0x73,0x00,0x75,0x6c,0x6f,0x6e,0x67,0x2a,0x00,0x72,0x6f,0x75,0x6e,0x64,0x69,0x6e,0x67,0x5f,0x6d,0x6f,0x64,0x65,0x73,0x00,0x75,0x69,0x6e,0x74,0x2a,0x00,0x70 +,0x72,0x6f,0x67,0x72,0x61,0x6d,0x73,0x00,0x75,0x69,0x6e,0x74,0x2a,0x00,0x62,0x61,0x74,0x63,0x68,0x5f,0x73,0x69,0x7a,0x65,0x00,0x75,0x69,0x6e,0x74,0x00,0x72,0x78 +,0x5f,0x70,0x61,0x72,0x61,0x6d,0x65,0x74,0x65,0x72,0x73,0x00,0x75,0x69,0x6e,0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x7f,0x45,0x4c,0x46,0x02,0x01,0x01,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0xe0,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x38,0x00,0x01,0x00,0x40,0x00,0x06,0x00,0x01 +,0x00,0x03,0x00,0x00,0x60,0x05,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0xb4,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0xb4,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2e,0x73,0x68,0x73,0x74,0x72 +,0x74,0x61,0x62,0x00,0x2e,0x73,0x74,0x72,0x74,0x61,0x62,0x00,0x2e,0x6e,0x6f,0x74,0x65,0x00,0x2e,0x68,0x73,0x61,0x74,0x65,0x78,0x74,0x00,0x2e,0x73,0x79,0x6d,0x74 +,0x61,0x62,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x26,0x5f,0x5f,0x4f,0x70,0x65,0x6e,0x43,0x4c,0x5f,0x72,0x61,0x6e,0x64,0x6f,0x6d,0x78,0x5f,0x72,0x75,0x6e,0x5f +,0x6b,0x65,0x72,0x6e,0x65,0x6c,0x00,0x5f,0x5f,0x68,0x73,0x61,0x5f,0x73,0x65,0x63,0x74,0x69,0x6f,0x6e,0x2e,0x68,0x73,0x61,0x74,0x65,0x78,0x74,0x00,0x00,0x00,0x00 +,0x00,0x04,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x4d,0x44,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x0c,0x00,0x00 +,0x00,0x02,0x00,0x00,0x00,0x41,0x4d,0x44,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x01,0x01,0x00,0x04,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x03,0x00,0x00 +,0x00,0x41,0x4d,0x44,0x00,0x04,0x00,0x07,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x4d,0x44,0x00,0x41,0x4d,0x44,0x47,0x50,0x55,0x00 +,0x00,0x04,0x00,0x00,0x00,0x29,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x41,0x4d,0x44,0x00,0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x4d,0x44 +,0x20,0x48,0x53,0x41,0x20,0x52,0x75,0x6e,0x74,0x69,0x6d,0x65,0x20,0x46,0x69,0x6e,0x61,0x6c,0x69,0x7a,0x65,0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00 +,0x00,0x1a,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x41,0x4d,0x44,0x00,0x16,0x00,0x2d,0x68,0x73,0x61,0x5f,0x63,0x61,0x6c,0x6c,0x5f,0x63,0x6f,0x6e,0x76,0x65,0x6e,0x74 +,0x69,0x6f,0x6e,0x3d,0x30,0x00,0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x5f,0x03,0xac,0x00,0x8c,0x00,0x00,0x00,0x09,0x00,0x0a,0x00,0x00,0x00,0x00 +,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x62,0x00,0x80,0x00,0x80,0x00,0x00,0x00,0x60,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x04,0x04,0x04,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0xff,0x00,0xfc,0xbe,0x00,0x00,0x01,0x00,0x00,0x00,0x84,0xc0,0x00,0x00,0x00,0x00,0x70,0x00,0x8c,0xbf,0x00,0x00,0x93,0xbf,0x00,0x00,0x82,0xbf,0x06,0x00,0x88 +,0xbe,0x01,0x00,0x12,0xd1,0x86,0x10,0x00,0x00,0x01,0x01,0x02,0x32,0x02,0x00,0x06,0xc0,0x00,0x00,0x00,0x00,0x82,0x00,0x06,0xc0,0x40,0x00,0x00,0x00,0x02,0x10,0x06 +,0xc0,0x48,0x00,0x00,0x00,0x7f,0x00,0x8c,0xbf,0x08,0x82,0x10,0x8e,0x40,0x10,0x40,0x80,0x41,0x80,0x41,0x82,0x40,0x02,0x10,0x7e,0x41,0x02,0x12,0x7e,0x00,0x00,0x50 +,0xdc,0x08,0x00,0x00,0x08,0x70,0x0f,0x8c,0xbf,0x42,0x00,0x89,0xd2,0x08,0x01,0x01,0x00,0x81,0x08,0x42,0xb9,0x80,0x00,0xc3,0xbe,0xff,0x00,0xc4,0xbe,0x00,0x01,0x00 +,0x00,0x80,0x00,0xc5,0xbe,0x00,0x02,0x02,0x32,0x86,0x02,0x04,0x20,0x85,0x04,0x06,0x24,0xbf,0x02,0x02,0x26,0x80,0x02,0x08,0x7e,0x03,0x00,0x8f,0xd2,0x83,0x06,0x02 +,0x00,0x84,0x02,0x0a,0x24,0x02,0x06,0x06,0x32,0x03,0x02,0x0c,0x7e,0x06,0x09,0x08,0x38,0x82,0x02,0x52,0x24,0x03,0x53,0x0c,0x32,0x07,0x6a,0x1c,0xd1,0x04,0x01,0xa9 +,0x01,0x00,0x00,0x50,0xdc,0x06,0x00,0x00,0x06,0x80,0x02,0x00,0x7e,0x70,0x0f,0x8c,0xbf,0x00,0x00,0x1a,0xd8,0x29,0x06,0x00,0x00,0x7f,0x00,0x8c,0xbf,0x7e,0x01,0x80 +,0xbe,0x02,0x00,0xdb,0xd0,0x01,0x0f,0x01,0x00,0x75,0x01,0x88,0xbf,0x02,0x05,0x02,0xc0,0x5c,0x00,0x00,0x00,0x7f,0x00,0x8c,0xbf,0x14,0xff,0x95,0x92,0x00,0x00,0x05 +,0x00,0x81,0x15,0x15,0x8e,0x14,0xff,0x96,0x92,0x05,0x00,0x05,0x00,0x81,0x16,0x16,0x8e,0x14,0xff,0x97,0x92,0x0a,0x00,0x05,0x00,0x81,0x17,0x17,0x8e,0x14,0xff,0x98 +,0x92,0x0f,0x00,0x04,0x00,0x81,0x18,0x18,0x8e,0x17,0xc0,0x02,0x80,0x14,0x00,0x86,0xd2,0x02,0x05,0x00,0x00,0x02,0x00,0x85,0xd2,0x02,0x05,0x00,0x00,0x80,0x02,0x52 +,0x7e,0x80,0x02,0x58,0x7e,0x98,0x00,0x6c,0xd8,0x00,0x00,0x00,0x06,0x02,0x00,0xc9,0xd0,0x01,0x09,0x01,0x00,0x12,0x10,0xee,0xd8,0x00,0x00,0x00,0x22,0x88,0x00,0xec +,0xd8,0x00,0x00,0x00,0x0b,0x00,0x00,0x09,0xb0,0x7e,0x01,0x86,0xbe,0x06,0x02,0xfe,0x89,0xa0,0x00,0xec,0xd8,0x00,0x00,0x00,0x0d,0x06,0x7e,0xfe,0x89,0x80,0x02,0x1a +,0x7e,0x80,0x02,0x1c,0x7e,0x06,0x01,0xfe,0xbe,0x08,0x01,0x86,0xbe,0x40,0x27,0x86,0xb7,0x00,0x0b,0x0a,0x32,0x05,0x6a,0x19,0xd1,0x05,0x81,0x01,0x00,0x7e,0x01,0x88 +,0xbe,0x08,0x02,0xfe,0x89,0xa8,0x00,0xec,0xd8,0x00,0x00,0x00,0x0f,0x08,0x7e,0xfe,0x89,0x80,0x02,0x1e,0x7e,0x80,0x02,0x20,0x7e,0x08,0x01,0xfe,0xbe,0x02,0x02,0x0a +,0xc0,0x30,0x00,0x00,0x00,0x02,0x04,0x02,0xc0,0x58,0x00,0x00,0x00,0x02,0x01,0x06,0xc0,0x50,0x00,0x00,0x00,0x83,0x02,0x02,0x24,0x00,0x03,0x22,0x32,0x7f,0x00,0x8c +,0xbf,0x0a,0x04,0x04,0x32,0x0b,0x02,0x24,0x7e,0x12,0x29,0x24,0x38,0xff,0x02,0x26,0x7e,0xff,0xff,0xff,0x00,0x08,0x0c,0x0c,0x32,0x09,0x02,0x28,0x7e,0x14,0x6a,0x1c +,0xd1,0x14,0x01,0xa9,0x01,0x00,0x00,0xec,0xd8,0x11,0x00,0x00,0x15,0x04,0x06,0x04,0x80,0x05,0x07,0x05,0x82,0x13,0x00,0x00,0xd1,0x13,0x83,0x09,0x00,0x83,0x46,0x10 +,0x24,0x83,0x44,0x0e,0x24,0x83,0x18,0x18,0x24,0x83,0x16,0x14,0x24,0x08,0x01,0x10,0x32,0x07,0x01,0x0e,0x32,0x0c,0x01,0x18,0x32,0x0a,0x01,0x00,0x32,0x24,0x03,0x14 +,0x7e,0x25,0x03,0x2e,0x7e,0x18,0x81,0x82,0x80,0x10,0x00,0x83,0xbe,0x26,0x6a,0x1a,0xd1,0x15,0x10,0x01,0x00,0x27,0x6a,0x1a,0xd1,0x16,0x10,0x01,0x00,0x32,0x6a,0x1a +,0xd1,0x17,0x10,0x01,0x00,0xff,0x02,0x66,0x7e,0x00,0x00,0xf0,0x80,0x03,0x03,0x68,0x7e,0x12,0x03,0x06,0x7e,0x34,0x03,0x24,0x7e,0x00,0x00,0x89,0xd2,0x02,0x01,0x01 +,0x00,0x01,0x00,0x89,0xd2,0x03,0x01,0x01,0x00,0x7e,0x01,0xa4,0xbe,0x82,0x01,0xfe,0xbe,0x88,0x02,0x52,0x7e,0x84,0x02,0x58,0x7e,0x83,0x01,0xfe,0xbe,0x18,0x1a,0xee +,0xd8,0x29,0x00,0x00,0x34,0x1c,0x1e,0xee,0xd8,0x29,0x00,0x00,0x38,0xff,0x02,0x9a,0x7e,0xff,0xff,0xff,0x00,0xa0,0x00,0xec,0xd8,0x29,0x00,0x00,0x4e,0x24,0x01,0xfe +,0xbe,0xff,0x02,0xa4,0x7e,0x00,0x00,0x00,0x80,0xff,0x02,0xa6,0x7e,0x00,0x00,0xf0,0x3f,0xff,0x02,0xa8,0x7e,0x00,0x00,0x10,0x00,0x00,0x1c,0x8e,0xbe,0x0e,0xff,0x28 +,0x80,0xe8,0x03,0x00,0x00,0x0f,0x80,0x29,0x82,0x0e,0xff,0x2a,0x80,0x4c,0x04,0x00,0x00,0x0f,0x80,0x2b,0x82,0x0e,0xff,0x2c,0x80,0xb0,0x04,0x00,0x00,0x0f,0x80,0x2d +,0x82,0x0e,0xff,0x2e,0x80,0x14,0x05,0x00,0x00,0x0f,0x80,0x2f,0x82,0x0e,0xff,0x30,0x80,0x78,0x05,0x00,0x00,0x0f,0x80,0x31,0x82,0x0e,0xff,0x32,0x80,0xe0,0x05,0x00 +,0x00,0x0f,0x80,0x33,0x82,0x0e,0xff,0x34,0x80,0x48,0x06,0x00,0x00,0x0f,0x80,0x35,0x82,0x0e,0xff,0x36,0x80,0xb0,0x06,0x00,0x00,0x0f,0x80,0x37,0x82,0x0e,0xff,0x38 +,0x80,0x18,0x07,0x00,0x00,0x0f,0x80,0x39,0x82,0x0e,0xff,0x3a,0x80,0x88,0x07,0x00,0x00,0x0f,0x80,0x3b,0x82,0xc1,0x00,0xbf,0xbe,0xff,0x00,0xc6,0xbe,0x00,0xff,0x00 +,0x00,0xff,0x00,0xc7,0xbe,0x00,0xfe,0x01,0x00,0xff,0x00,0xc8,0xbe,0x00,0xfc,0x03,0x00,0xff,0x00,0xc9,0xbe,0x00,0xf8,0x07,0x00,0xff,0x00,0xca,0xbe,0x00,0xf0,0x0f +,0x00,0xff,0x00,0xcb,0xbe,0x00,0xe0,0x1f,0x00,0xff,0x00,0xcc,0xbe,0x00,0xc0,0x3f,0x00,0xff,0x00,0xcd,0xbe,0x00,0x80,0x7f,0x00,0xff,0x00,0xce,0xbe,0x00,0x00,0xff +,0x00,0xff,0x00,0xcf,0xbe,0x00,0x00,0xfe,0x01,0xff,0x00,0xd0,0xbe,0x00,0x00,0xfc,0x03,0xff,0x00,0xd1,0xbe,0x00,0x00,0xf8,0x07,0xff,0x00,0xd2,0xbe,0x00,0x00,0xf0 +,0x0f,0xff,0x00,0xd3,0xbe,0x00,0x00,0xe0,0x1f,0xff,0x00,0xd4,0xbe,0x00,0x00,0xc0,0x3f,0xff,0x00,0xd5,0xbe,0x00,0x00,0x80,0x7f,0x17,0xc0,0xd6,0x80,0x00,0x00,0xec +,0xd8,0x00,0x00,0x00,0x18,0x00,0x00,0xec,0xd8,0x0c,0x00,0x00,0x1a,0x7f,0x00,0x8c,0xbf,0x1b,0x33,0x32,0x2a,0x1a,0x31,0x30,0x2a,0x19,0x15,0x14,0x2a,0x18,0x2f,0x2e +,0x2a,0x56,0x14,0x14,0x26,0x56,0x2e,0x2e,0x26,0x0a,0x03,0x14,0x32,0x17,0x03,0x2e,0x32,0x02,0x15,0x34,0x32,0x1b,0x6a,0x1c,0xd1,0x03,0x01,0xa9,0x01,0x02,0x2f,0x2e +,0x32,0x18,0x6a,0x1c,0xd1,0x03,0x01,0xa9,0x01,0x00,0x00,0x54,0xdc,0x1a,0x00,0x00,0x1c,0x00,0x00,0x54,0xdc,0x17,0x00,0x00,0x1e,0x71,0x0f,0x8c,0xbf,0x1c,0x09,0x40 +,0x7e,0x1d,0x09,0x38,0x7e,0x70,0x0f,0x8c,0xbf,0x15,0x3d,0x44,0x2a,0x16,0x3f,0x46,0x2a,0x06,0x49,0x2c,0x32,0x19,0x6a,0x1c,0xd1,0x14,0x01,0xa9,0x01,0x16,0x03,0x2a +,0x32,0x16,0x6a,0x1c,0xd1,0x19,0x01,0xa9,0x01,0x00,0x00,0x54,0xdc,0x15,0x00,0x00,0x15,0x20,0x1b,0x3c,0x28,0x21,0x27,0x3e,0x26,0x1f,0x1d,0x3e,0x28,0x1c,0x1f,0x38 +,0x28,0x1d,0x27,0x3a,0x26,0x1d,0x21,0x3a,0x28,0x00,0x01,0x9c,0xd8,0x05,0x1e,0x1c,0x00,0x7f,0x00,0x8c,0xbf,0x83,0x01,0xfe,0xbe,0x08,0x0a,0xee,0xd8,0x29,0x00,0x00 +,0x3c,0x0c,0x0e,0xee,0xd8,0x29,0x00,0x00,0x40,0x10,0x12,0xee,0xd8,0x29,0x00,0x00,0x44,0x14,0x16,0xee,0xd8,0x29,0x00,0x00,0x48,0x10,0x00,0x89,0xd2,0x22,0x01,0x01 +,0x00,0x11,0x00,0x89,0xd2,0x23,0x01,0x01,0x00,0x12,0x00,0x89,0xd2,0x22,0x03,0x01,0x00,0x13,0x00,0x89,0xd2,0x23,0x03,0x01,0x00,0x14,0x00,0x89,0xd2,0x22,0x05,0x01 +,0x00,0x15,0x00,0x89,0xd2,0x23,0x05,0x01,0x00,0x16,0x00,0x89,0xd2,0x22,0x07,0x01,0x00,0x17,0x00,0x89,0xd2,0x23,0x07,0x01,0x00,0x18,0x00,0x89,0xd2,0x22,0x09,0x01 +,0x00,0x19,0x00,0x89,0xd2,0x23,0x09,0x01,0x00,0x1a,0x00,0x89,0xd2,0x22,0x0b,0x01,0x00,0x1b,0x00,0x89,0xd2,0x23,0x0b,0x01,0x00,0x1c,0x00,0x89,0xd2,0x22,0x0d,0x01 +,0x00,0x1d,0x00,0x89,0xd2,0x23,0x0d,0x01,0x00,0x1e,0x00,0x89,0xd2,0x22,0x0f,0x01,0x00,0x1f,0x00,0x89,0xd2,0x23,0x0f,0x01,0x00,0x7f,0x00,0x8c,0xbf,0x04,0x1e,0x8c +,0xbe,0x08,0x0a,0x9c,0xd8,0x29,0x3c,0x3e,0x00,0x0c,0x0e,0x9c,0xd8,0x29,0x40,0x42,0x00,0x10,0x12,0x9c,0xd8,0x29,0x44,0x46,0x00,0x14,0x16,0x9c,0xd8,0x29,0x48,0x4a +,0x00,0x1c,0x00,0x8a,0xd2,0x10,0x00,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x11,0x00,0x01,0x00,0x1c,0x00,0x8a,0xd2,0x12,0x02,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x13,0x02,0x01 +,0x00,0x1c,0x00,0x8a,0xd2,0x14,0x04,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x15,0x04,0x01,0x00,0x1c,0x00,0x8a,0xd2,0x16,0x06,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x17,0x06,0x01 +,0x00,0x1c,0x00,0x8a,0xd2,0x18,0x08,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x19,0x08,0x01,0x00,0x1c,0x00,0x8a,0xd2,0x1a,0x0a,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x1b,0x0a,0x01 +,0x00,0x1c,0x00,0x8a,0xd2,0x1c,0x0c,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x1d,0x0c,0x01,0x00,0x1c,0x00,0x8a,0xd2,0x1e,0x0e,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x1f,0x0e,0x01 +,0x00,0x24,0x01,0xfe,0xbe,0x00,0x00,0x9a,0xd8,0x11,0x1c,0x00,0x00,0x7f,0x00,0x8c,0xbf,0x1c,0x2b,0x2a,0x2a,0x1d,0x2d,0x2c,0x2a,0x00,0x00,0x6c,0xd8,0x07,0x00,0x00 +,0x1c,0x00,0x00,0x6c,0xd8,0x08,0x00,0x00,0x1d,0x00,0x00,0x9a,0xd8,0x11,0x15,0x00,0x00,0x7f,0x01,0x8c,0xbf,0x08,0x10,0xee,0xd8,0x11,0x00,0x00,0x1e,0x1c,0x4b,0x14 +,0x2a,0x7f,0x00,0x8c,0xbf,0x20,0x3d,0x3c,0x2a,0x21,0x3f,0x3e,0x2a,0x0a,0x3b,0x14,0x2a,0x00,0x00,0x74,0xdc,0x1a,0x15,0x00,0x00,0xff,0x14,0x14,0x26,0xc0,0xff,0xff +,0x7f,0x00,0x00,0x74,0xdc,0x17,0x1e,0x00,0x00,0x02,0x80,0x06,0xbf,0x06,0x00,0x85,0xbf,0x02,0x81,0x82,0x81,0x24,0x03,0x4a,0x7e,0x80,0x02,0x2e,0x7e,0x0a,0x03,0x48 +,0x7e,0x80,0x02,0x14,0x7e,0x5d,0xff,0x82,0xbf,0x12,0x03,0x00,0x32,0x01,0x6a,0x1c,0xd1,0x04,0x01,0xa9,0x01,0x00,0x00,0x74,0xdc,0x00,0x15,0x00,0x00,0x00,0x6a,0x19 +,0xd1,0x00,0x81,0x01,0x00,0x01,0x6a,0x1c,0xd1,0x01,0x01,0xa9,0x01,0x00,0x00,0x74,0xdc,0x00,0x1e,0x00,0x00,0x00,0x6a,0x19,0xd1,0x00,0x81,0x01,0x00,0x01,0x6a,0x1c +,0xd1,0x01,0x01,0xa9,0x01,0x00,0x00,0x74,0xdc,0x00,0x20,0x00,0x00,0x40,0x02,0x00,0x7e,0x41,0x02,0x02,0x7e,0x42,0x02,0x04,0x7e,0x00,0x00,0x70,0xdc,0x00,0x02,0x00 +,0x00,0x00,0x00,0x81,0xbf,0x81,0x08,0x43,0xb9,0x44,0x4d,0x38,0x7e,0x2a,0x00,0x81,0xd2,0x1c,0x89,0x02,0x00,0x1c,0x03,0x60,0x7e,0x1d,0xa9,0x62,0x34,0x1c,0x03,0x5c +,0x7e,0x31,0xa5,0x5e,0x2a,0x2e,0x00,0xcc,0xd1,0x2e,0x55,0xc2,0x03,0x2a,0x00,0xcc,0xd1,0x2a,0x5d,0xaa,0x04,0x30,0x00,0xcc,0xd1,0x30,0x5d,0xc2,0x04,0x2e,0x00,0xcc +,0xd1,0x2a,0x55,0x12,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2e,0x61,0xaa,0x04,0x0e,0x00,0x13,0xd0,0x44,0x89,0x00,0x00,0x2a,0x03,0x88,0x7e,0x2b,0x03,0x8a +,0x7e,0x83,0x01,0xfe,0xbe,0x3c,0x1d,0x80,0xbe,0x81,0x08,0x43,0xb9,0x46,0x4d,0x38,0x7e,0x2a,0x00,0x81,0xd2,0x1c,0x8d,0x02,0x00,0x1c,0x03,0x60,0x7e,0x1d,0xa9,0x62 +,0x34,0x1c,0x03,0x5c,0x7e,0x31,0xa5,0x5e,0x2a,0x2e,0x00,0xcc,0xd1,0x2e,0x55,0xc2,0x03,0x2a,0x00,0xcc,0xd1,0x2a,0x5d,0xaa,0x04,0x30,0x00,0xcc,0xd1,0x30,0x5d,0xc2 +,0x04,0x2e,0x00,0xcc,0xd1,0x2a,0x55,0x1a,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2e,0x61,0xaa,0x04,0x0e,0x00,0x13,0xd0,0x46,0x89,0x00,0x00,0x2a,0x03,0x8c +,0x7e,0x2b,0x03,0x8e,0x7e,0x83,0x01,0xfe,0xbe,0x3c,0x1d,0x80,0xbe,0x81,0x08,0x43,0xb9,0x48,0x4d,0x38,0x7e,0x2a,0x00,0x81,0xd2,0x1c,0x91,0x02,0x00,0x1c,0x03,0x60 +,0x7e,0x1d,0xa9,0x62,0x34,0x1c,0x03,0x5c,0x7e,0x31,0xa5,0x5e,0x2a,0x2e,0x00,0xcc,0xd1,0x2e,0x55,0xc2,0x03,0x2a,0x00,0xcc,0xd1,0x2a,0x5d,0xaa,0x04,0x30,0x00,0xcc +,0xd1,0x30,0x5d,0xc2,0x04,0x2e,0x00,0xcc,0xd1,0x2a,0x55,0x22,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2e,0x61,0xaa,0x04,0x0e,0x00,0x13,0xd0,0x48,0x89,0x00 +,0x00,0x2a,0x03,0x90,0x7e,0x2b,0x03,0x92,0x7e,0x83,0x01,0xfe,0xbe,0x3c,0x1d,0x80,0xbe,0x81,0x08,0x43,0xb9,0x4a,0x4d,0x38,0x7e,0x2a,0x00,0x81,0xd2,0x1c,0x95,0x02 +,0x00,0x1c,0x03,0x60,0x7e,0x1d,0xa9,0x62,0x34,0x1c,0x03,0x5c,0x7e,0x31,0xa5,0x5e,0x2a,0x2e,0x00,0xcc,0xd1,0x2e,0x55,0xc2,0x03,0x2a,0x00,0xcc,0xd1,0x2a,0x5d,0xaa +,0x04,0x30,0x00,0xcc,0xd1,0x30,0x5d,0xc2,0x04,0x2e,0x00,0xcc,0xd1,0x2a,0x55,0x2a,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2e,0x61,0xaa,0x04,0x0e,0x00,0x13 +,0xd0,0x4a,0x89,0x00,0x00,0x2a,0x03,0x94,0x7e,0x2b,0x03,0x96,0x7e,0x83,0x01,0xfe,0xbe,0x3c,0x1d,0x80,0xbe,0x1c,0x9d,0x38,0x28,0x1d,0x9b,0x3a,0x26,0x1d,0x9f,0x3a +,0x28,0x81,0x08,0x43,0xb9,0x1c,0x4b,0x60,0x7e,0x50,0x00,0xcc,0xd1,0x1c,0x61,0xca,0x23,0x30,0x00,0xcc,0xd1,0x30,0xa1,0xc2,0x04,0x50,0x00,0x81,0xd2,0x44,0x61,0x02 +,0x00,0x2a,0x00,0xcc,0xd1,0x1c,0xa1,0x12,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2a,0x61,0x42,0x05,0x50,0x00,0xdf,0xd1,0x2a,0x39,0x12,0x05,0x0e,0x00,0x72 +,0xd0,0x44,0x39,0x02,0x00,0x80,0x02,0xa0,0x7e,0x53,0x03,0xa2,0x7e,0x83,0x01,0xfe,0xbe,0x50,0x03,0x88,0x7e,0x51,0x03,0x8a,0x7e,0x3c,0x1d,0x80,0xbe,0x1c,0x9d,0x38 +,0x28,0x1d,0x9b,0x3a,0x26,0x1d,0x9f,0x3a,0x28,0x81,0x08,0x43,0xb9,0x1c,0x4b,0x60,0x7e,0x50,0x00,0xcc,0xd1,0x1c,0x61,0xca,0x23,0x30,0x00,0xcc,0xd1,0x30,0xa1,0xc2 +,0x04,0x50,0x00,0x81,0xd2,0x46,0x61,0x02,0x00,0x2a,0x00,0xcc,0xd1,0x1c,0xa1,0x1a,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2a,0x61,0x42,0x05,0x50,0x00,0xdf +,0xd1,0x2a,0x39,0x1a,0x05,0x0e,0x00,0x72,0xd0,0x46,0x39,0x02,0x00,0x80,0x02,0xa0,0x7e,0x53,0x03,0xa2,0x7e,0x83,0x01,0xfe,0xbe,0x50,0x03,0x8c,0x7e,0x51,0x03,0x8e +,0x7e,0x3c,0x1d,0x80,0xbe,0x1c,0x9d,0x38,0x28,0x1d,0x9b,0x3a,0x26,0x1d,0x9f,0x3a,0x28,0x81,0x08,0x43,0xb9,0x1c,0x4b,0x60,0x7e,0x50,0x00,0xcc,0xd1,0x1c,0x61,0xca +,0x23,0x30,0x00,0xcc,0xd1,0x30,0xa1,0xc2,0x04,0x50,0x00,0x81,0xd2,0x48,0x61,0x02,0x00,0x2a,0x00,0xcc,0xd1,0x1c,0xa1,0x22,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc +,0xd1,0x2a,0x61,0x42,0x05,0x50,0x00,0xdf,0xd1,0x2a,0x39,0x22,0x05,0x0e,0x00,0x72,0xd0,0x48,0x39,0x02,0x00,0x80,0x02,0xa0,0x7e,0x53,0x03,0xa2,0x7e,0x83,0x01,0xfe +,0xbe,0x50,0x03,0x90,0x7e,0x51,0x03,0x92,0x7e,0x3c,0x1d,0x80,0xbe,0x1c,0x9d,0x38,0x28,0x1d,0x9b,0x3a,0x26,0x1d,0x9f,0x3a,0x28,0x81,0x08,0x43,0xb9,0x1c,0x4b,0x60 +,0x7e,0x50,0x00,0xcc,0xd1,0x1c,0x61,0xca,0x23,0x30,0x00,0xcc,0xd1,0x30,0xa1,0xc2,0x04,0x50,0x00,0x81,0xd2,0x4a,0x61,0x02,0x00,0x2a,0x00,0xcc,0xd1,0x1c,0xa1,0x2a +,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2a,0x61,0x42,0x05,0x50,0x00,0xdf,0xd1,0x2a,0x39,0x2a,0x05,0x0e,0x00,0x72,0xd0,0x4a,0x39,0x02,0x00,0x80,0x02,0xa0 +,0x7e,0x53,0x03,0xa2,0x7e,0x83,0x01,0xfe,0xbe,0x50,0x03,0x94,0x7e,0x51,0x03,0x96,0x7e,0x3c,0x1d,0x80,0xbe,0x81,0x01,0xfe,0xbe,0x0e,0x02,0x5a,0x7e,0x28,0x00,0x86 +,0xd2,0x26,0x5a,0x02,0x00,0x0f,0x02,0x5e,0x7e,0x2a,0x20,0xe8,0xd1,0x26,0x5e,0xa2,0x04,0x2a,0x03,0x50,0x7e,0x2d,0x20,0xe8,0xd1,0x27,0x5a,0xa2,0x04,0x2a,0x20,0xe8 +,0xd1,0x27,0x5e,0xae,0x04,0x2a,0x5d,0x54,0x32,0x80,0x56,0x56,0x38,0x20,0x00,0x89,0xd2,0x2a,0x01,0x01,0x00,0x21,0x00,0x89,0xd2,0x2b,0x01,0x01,0x00,0x0f,0x80,0x04 +,0xbf,0x26,0x80,0xa2,0x85,0x20,0x22,0xa0,0x80,0x21,0x23,0xa1,0x82,0x27,0x80,0x04,0xbf,0x0e,0x80,0xa2,0x85,0x20,0x22,0x8e,0x80,0x21,0x23,0x8f,0x82,0x83,0x01,0xfe +,0xbe,0x3c,0x1d,0x80,0xbe,0x81,0x01,0xfe,0xbe,0x26,0x02,0x5a,0x7e,0x28,0x00,0x86,0xd2,0x0e,0x5a,0x02,0x00,0x27,0x02,0x5e,0x7e,0x2a,0x20,0xe8,0xd1,0x0e,0x5e,0xa2 +,0x04,0x2a,0x03,0x50,0x7e,0x2d,0x20,0xe8,0xd1,0x0f,0x5a,0xa2,0x04,0x2a,0x20,0xe8,0xd1,0x0f,0x5e,0xae,0x04,0x2a,0x5d,0x54,0x32,0x80,0x56,0x56,0x38,0x0e,0x00,0x89 +,0xd2,0x2a,0x01,0x01,0x00,0x0f,0x00,0x89,0xd2,0x2b,0x01,0x01,0x00,0x83,0x01,0xfe,0xbe,0x3c,0x1d,0x80,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1a,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0xb4,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x03,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x2a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x0b,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x34,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x13,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0xc8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00 +,0x00,0xb4,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x22,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xb8,0x0d,0x00,0x00,0x00,0x00,0x00 +,0x00,0x48,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x0b,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x72 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x43,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x13,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xb8 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1b,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x24,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x27 +,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x3b,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2c,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x62 +,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x80,0x0f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; +const int randomx_run_gfx803_bin_size=6568; diff --git a/src/backend/opencl/cl/rx/randomx_run_gfx900.asm b/src/backend/opencl/cl/rx/randomx_run_gfx900.asm new file mode 100644 index 00000000..058b0d18 --- /dev/null +++ b/src/backend/opencl/cl/rx/randomx_run_gfx900.asm @@ -0,0 +1,688 @@ +/* +Copyright (c) 2019 SChernykh + +This file is part of RandomX OpenCL. + +RandomX OpenCL is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +RandomX OpenCL is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with RandomX OpenCL. If not, see . +*/ + +.amdcl2 +.gpu GFX900 +.64bit +.arch_minor 0 +.arch_stepping 0 +.driver_version 223600 +.kernel randomx_run + .config + .dims x + .cws 64, 1, 1 + .sgprsnum 96 + # 6 waves per SIMD: 37-40 VGPRs + # 5 waves per SIMD: 41-48 VGPRs + # 4 waves per SIMD: 49-64 VGPRs + # 3 waves per SIMD: 65-84 VGPRs + # 2 waves per SIMD: 85-128 VGPRs + # 1 wave per SIMD: 129-256 VGPRs + .vgprsnum 128 + .localsize 256 + .floatmode 0xc0 + .pgmrsrc1 0x00ac035f + .pgmrsrc2 0x00000090 + .dx10clamp + .ieeemode + .useargs + .priority 0 + .arg _.global_offset_0, "size_t", long + .arg _.global_offset_1, "size_t", long + .arg _.global_offset_2, "size_t", long + .arg _.printf_buffer, "size_t", void*, global, , rdonly + .arg _.vqueue_pointer, "size_t", long + .arg _.aqlwrap_pointer, "size_t", long + .arg dataset, "uchar*", uchar*, global, const, rdonly + .arg scratchpad, "uchar*", uchar*, global, + .arg registers, "ulong*", ulong*, global, + .arg rounding_modes, "uint*", uint*, global, + .arg programs, "uint*", uint*, global, + .arg batch_size, "uint", uint + .arg rx_parameters, "uint", uint + .text + s_mov_b32 m0, 0x10000 + s_dcache_wb + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_icache_inv + s_branch begin + + # pgmrsrc2 = 0x00000090, bits 1:5 = 8, so first 8 SGPRs (s0-s7) contain user data + # s8 contains group id + # v0 contains local id +begin: + v_lshl_add_u32 v1, s8, 6, v0 + s_load_dwordx2 s[0:1], s[4:5], 0x0 + s_load_dwordx2 s[2:3], s[4:5], 0x40 + s_load_dwordx2 s[64:65], s[4:5], 0x48 + s_waitcnt lgkmcnt(0) + + # load rounding mode + s_lshl_b32 s16, s8, 2 + s_add_u32 s64, s64, s16 + s_addc_u32 s65, s65, 0 + v_mov_b32 v8, 0 + global_load_dword v8, v8, s[64:65] + s_waitcnt vmcnt(0) + v_readlane_b32 s66, v8, 0 + s_setreg_b32 hwreg(mode, 2, 2), s66 + s_mov_b32 s67, 0 + + # used in FSQRT_R to check for "positive normal value" (v_cmpx_class_f64) + s_mov_b32 s68, 256 + s_mov_b32 s69, 0 + + v_add_u32 v1, s0, v1 + v_lshrrev_b32 v2, 6, v1 + v_lshlrev_b32 v3, 5, v2 + v_and_b32 v1, 63, v1 + v_mov_b32 v4, 0 + v_lshlrev_b64 v[3:4], 3, v[3:4] + v_lshlrev_b32 v5, 4, v1 + v_add_co_u32 v3, vcc, s2, v3 + v_mov_b32 v6, s3 + v_addc_co_u32 v4, vcc, v6, v4, vcc + v_lshlrev_b32 v41, 2, v1 + v_add_co_u32 v6, vcc, v3, v41 + v_addc_co_u32 v7, vcc, v4, 0, vcc + global_load_dword v6, v[6:7], off + v_mov_b32 v0, 0 + s_waitcnt vmcnt(0) + ds_write_b32 v41, v6 + s_waitcnt lgkmcnt(0) + s_mov_b64 s[0:1], exec + v_cmpx_le_u32 s[2:3], v1, 7 + s_cbranch_execz program_end + + # rx_parameters + s_load_dword s20, s[4:5], 0x5c + s_waitcnt lgkmcnt(0) + + # Scratchpad L1 size + s_bfe_u32 s21, s20, 0x050000 + s_lshl_b32 s21, 1, s21 + + # Scratchpad L2 size + s_bfe_u32 s22, s20, 0x050005 + s_lshl_b32 s22, 1, s22 + + # Scratchpad L3 size + s_bfe_u32 s23, s20, 0x05000A + s_lshl_b32 s23, 1, s23 + + # program iterations + s_bfe_u32 s24, s20, 0x04000F + s_lshl_b32 s24, 1, s24 + + # Base address for scratchpads + s_add_u32 s2, s23, 64 + v_mul_hi_u32 v20, v2, s2 + v_mul_lo_u32 v2, v2, s2 + + # v41, v44 = 0 + v_mov_b32 v41, 0 + v_mov_b32 v44, 0 + + ds_read_b32 v6, v0 offset:152 + v_cmp_lt_u32 s[2:3], v1, 4 + ds_read2_b64 v[34:37], v0 offset0:18 offset1:16 + ds_read_b64 v[11:12], v0 offset:136 + s_movk_i32 s9, 0x0 + s_mov_b64 s[6:7], exec + s_andn2_b64 exec, s[6:7], s[2:3] + ds_read_b64 v[13:14], v0 offset:160 + s_andn2_b64 exec, s[6:7], exec + v_mov_b32 v13, 0 + v_mov_b32 v14, 0 + s_mov_b64 exec, s[6:7] + + # compiled program size + s_mov_b64 s[6:7], s[8:9] + s_mulk_i32 s6, 10048 + + v_add3_u32 v5, v0, v5, 64 + s_mov_b64 s[8:9], exec + s_andn2_b64 exec, s[8:9], s[2:3] + ds_read_b64 v[15:16], v0 offset:168 + s_andn2_b64 exec, s[8:9], exec + v_mov_b32 v15, 0 + v_mov_b32 v16, 0 + s_mov_b64 exec, s[8:9] + s_load_dwordx4 s[8:11], s[4:5], 0x30 + + # batch_size + s_load_dword s16, s[4:5], 0x58 + + s_load_dwordx2 s[4:5], s[4:5], 0x50 + v_lshlrev_b32 v1, 3, v1 + v_add_u32 v17, v0, v1 + s_waitcnt lgkmcnt(0) + v_add_co_u32 v2, vcc, s10, v2 + v_mov_b32 v18, s11 + v_addc_co_u32 v18, vcc, v18, v20, vcc + v_mov_b32 v19, 0xffffff + v_add_co_u32 v6, vcc, s8, v6 + v_mov_b32 v20, s9 + v_addc_co_u32 v20, vcc, v20, 0, vcc + ds_read_b64 v[21:22], v17 + s_add_u32 s4, s4, s6 + s_addc_u32 s5, s5, s7 + v_cndmask_b32 v19, v19, -1, s[2:3] + v_lshl_add_u32 v8, v35, 3, v0 + v_lshl_add_u32 v7, v34, 3, v0 + v_lshl_add_u32 v12, v12, 3, v0 + v_lshl_add_u32 v0, v11, 3, v0 + v_mov_b32 v10, v36 + v_mov_b32 v23, v37 + + # loop counter + s_sub_u32 s2, s24, 1 + + # batch_size + s_mov_b32 s3, s16 + + # Scratchpad masks for scratchpads + v_sub_u32 v38, s21, 8 + v_sub_u32 v39, s22, 8 + v_sub_u32 v50, s23, 8 + + # mask for FSCAL_R + v_mov_b32 v51, 0x80F00000 + + # load scratchpad base address + v_readlane_b32 s0, v2, 0 + v_readlane_b32 s1, v18, 0 + + # save current executiom mask + s_mov_b64 s[36:37], exec + + # v41 = 0 on lane 0, set it to 8 on lane 1 + # v44 = 0 on lane 0, set it to 4 on lane 1 + s_mov_b64 exec, 2 + v_mov_b32 v41, 8 + v_mov_b32 v44, 4 + + # load group A registers + # Read low 8 bytes into lane 0 and high 8 bytes into lane 1 + s_mov_b64 exec, 3 + ds_read2_b64 v[52:55], v41 offset0:24 offset1:26 + ds_read2_b64 v[56:59], v41 offset0:28 offset1:30 + + # xmantissaMask + v_mov_b32 v77, (1 << 24) - 1 + + # xexponentMask + ds_read_b64 v[78:79], v41 offset:160 + + # Restore execution mask + s_mov_b64 exec, s[36:37] + + # sign mask (used in FSQRT_R) + v_mov_b32 v82, 0x80000000 + + # High 32 bits of "1.0" constant (used in FDIV_M) + v_mov_b32 v83, (1023 << 20) + + # Used to multiply FP64 values by 0.5 + v_mov_b32 v84, (1 << 20) + + s_getpc_b64 s[14:15] +cur_addr: + + # get addresses of FSQRT_R subroutines + s_add_u32 s40, s14, fsqrt_r_sub0 - cur_addr + s_addc_u32 s41, s15, 0 + s_add_u32 s42, s14, fsqrt_r_sub1 - cur_addr + s_addc_u32 s43, s15, 0 + s_add_u32 s44, s14, fsqrt_r_sub2 - cur_addr + s_addc_u32 s45, s15, 0 + s_add_u32 s46, s14, fsqrt_r_sub3 - cur_addr + s_addc_u32 s47, s15, 0 + + # get addresses of FDIV_M subroutines + s_add_u32 s48, s14, fdiv_m_sub0 - cur_addr + s_addc_u32 s49, s15, 0 + s_add_u32 s50, s14, fdiv_m_sub1 - cur_addr + s_addc_u32 s51, s15, 0 + s_add_u32 s52, s14, fdiv_m_sub2 - cur_addr + s_addc_u32 s53, s15, 0 + s_add_u32 s54, s14, fdiv_m_sub3 - cur_addr + s_addc_u32 s55, s15, 0 + + # get address for ISMULH_R subroutine + s_add_u32 s56, s14, ismulh_r_sub - cur_addr + s_addc_u32 s57, s15, 0 + + # get address for IMULH_R subroutine + s_add_u32 s58, s14, imulh_r_sub - cur_addr + s_addc_u32 s59, s15, 0 + + # used in IXOR_R instruction + s_mov_b32 s63, -1 + + # used in CBRANCH instruction + s_mov_b32 s70, (0xFF << 8) + s_mov_b32 s71, (0xFF << 9) + s_mov_b32 s72, (0xFF << 10) + s_mov_b32 s73, (0xFF << 11) + s_mov_b32 s74, (0xFF << 12) + s_mov_b32 s75, (0xFF << 13) + s_mov_b32 s76, (0xFF << 14) + s_mov_b32 s77, (0xFF << 15) + s_mov_b32 s78, (0xFF << 16) + s_mov_b32 s79, (0xFF << 17) + s_mov_b32 s80, (0xFF << 18) + s_mov_b32 s81, (0xFF << 19) + s_mov_b32 s82, (0xFF << 20) + s_mov_b32 s83, (0xFF << 21) + s_mov_b32 s84, (0xFF << 22) + s_mov_b32 s85, (0xFF << 23) + + # ScratchpadL3Mask64 + s_sub_u32 s86, s23, 64 + +main_loop: + # const uint2 spMix = as_uint2(R[readReg0] ^ R[readReg1]); + ds_read_b64 v[24:25], v0 + ds_read_b64 v[26:27], v12 + s_waitcnt lgkmcnt(0) + v_xor_b32 v25, v27, v25 + v_xor_b32 v24, v26, v24 + + # spAddr1 ^= spMix.y; + # spAddr0 ^= spMix.x; + v_xor_b32 v10, v25, v10 + v_xor_b32 v23, v24, v23 + + # spAddr1 &= ScratchpadL3Mask64; + # spAddr0 &= ScratchpadL3Mask64; + v_and_b32 v10, s86, v10 + v_and_b32 v23, s86, v23 + + # Offset for scratchpads + # offset1 = spAddr1 + sub * 8 + # offset0 = spAddr0 + sub * 8 + v_add_u32 v10, v10, v1 + v_add_u32 v23, v23, v1 + + # __global ulong* p1 = (__global ulong*)(scratchpad + offset1); + # __global ulong* p0 = (__global ulong*)(scratchpad + offset0); + v_add_co_u32 v26, vcc, v2, v10 + v_addc_co_u32 v27, vcc, v18, 0, vcc + v_add_co_u32 v23, vcc, v2, v23 + v_addc_co_u32 v24, vcc, v18, 0, vcc + + # load from spAddr1 + global_load_dwordx2 v[28:29], v[26:27], off + + # load from spAddr0 + global_load_dwordx2 v[30:31], v[23:24], off + s_waitcnt vmcnt(1) + + v_cvt_f64_i32 v[32:33], v28 + v_cvt_f64_i32 v[28:29], v29 + s_waitcnt vmcnt(0) + + # R[sub] ^= *p0; + v_xor_b32 v34, v21, v30 + v_xor_b32 v35, v22, v31 + + v_add_co_u32 v22, vcc, v6, v36 + v_addc_co_u32 v25, vcc, v20, 0, vcc + v_add_co_u32 v21, vcc, v22, v1 + v_addc_co_u32 v22, vcc, v25, 0, vcc + global_load_dwordx2 v[21:22], v[21:22], off + v_or_b32 v30, v32, v13 + v_and_or_b32 v31, v33, v19, v14 + v_or_b32 v28, v28, v15 + v_and_or_b32 v29, v29, v19, v16 + ds_write2_b64 v5, v[30:31], v[28:29] offset1:1 + s_waitcnt lgkmcnt(0) + + # Program 0 + + # load group F,E registers + # Read low 8 bytes into lane 0 and high 8 bytes into lane 1 + s_mov_b64 exec, 3 + ds_read2_b64 v[60:63], v41 offset0:8 offset1:10 + ds_read2_b64 v[64:67], v41 offset0:12 offset1:14 + ds_read2_b64 v[68:71], v41 offset0:16 offset1:18 + ds_read2_b64 v[72:75], v41 offset0:20 offset1:22 + + # load VM integer registers + v_readlane_b32 s16, v34, 0 + v_readlane_b32 s17, v35, 0 + v_readlane_b32 s18, v34, 1 + v_readlane_b32 s19, v35, 1 + v_readlane_b32 s20, v34, 2 + v_readlane_b32 s21, v35, 2 + v_readlane_b32 s22, v34, 3 + v_readlane_b32 s23, v35, 3 + v_readlane_b32 s24, v34, 4 + v_readlane_b32 s25, v35, 4 + v_readlane_b32 s26, v34, 5 + v_readlane_b32 s27, v35, 5 + v_readlane_b32 s28, v34, 6 + v_readlane_b32 s29, v35, 6 + v_readlane_b32 s30, v34, 7 + v_readlane_b32 s31, v35, 7 + + s_waitcnt lgkmcnt(0) + + # call JIT code + s_swappc_b64 s[12:13], s[4:5] + + # Write out group F,E registers + # Write low 8 bytes from lane 0 and high 8 bytes from lane 1 + ds_write2_b64 v41, v[60:61], v[62:63] offset0:8 offset1:10 + ds_write2_b64 v41, v[64:65], v[66:67] offset0:12 offset1:14 + ds_write2_b64 v41, v[68:69], v[70:71] offset0:16 offset1:18 + ds_write2_b64 v41, v[72:73], v[74:75] offset0:20 offset1:22 + + # store VM integer registers + v_writelane_b32 v28, s16, 0 + v_writelane_b32 v29, s17, 0 + v_writelane_b32 v28, s18, 1 + v_writelane_b32 v29, s19, 1 + v_writelane_b32 v28, s20, 2 + v_writelane_b32 v29, s21, 2 + v_writelane_b32 v28, s22, 3 + v_writelane_b32 v29, s23, 3 + v_writelane_b32 v28, s24, 4 + v_writelane_b32 v29, s25, 4 + v_writelane_b32 v28, s26, 5 + v_writelane_b32 v29, s27, 5 + v_writelane_b32 v28, s28, 6 + v_writelane_b32 v29, s29, 6 + v_writelane_b32 v28, s30, 7 + v_writelane_b32 v29, s31, 7 + + # Restore execution mask + s_mov_b64 exec, s[36:37] + + # Write out VM integer registers + ds_write_b64 v17, v[28:29] + + s_waitcnt lgkmcnt(0) + v_xor_b32 v21, v28, v21 + v_xor_b32 v22, v29, v22 + ds_read_b32 v28, v7 + ds_read_b32 v29, v8 + ds_write_b64 v17, v[21:22] + s_waitcnt lgkmcnt(1) + ds_read2_b64 v[30:33], v17 offset0:8 offset1:16 + v_xor_b32 v10, v28, v37 + s_waitcnt lgkmcnt(0) + v_xor_b32 v30, v32, v30 + v_xor_b32 v31, v33, v31 + v_xor_b32 v10, v10, v29 + global_store_dwordx2 v[26:27], v[21:22], off + v_and_b32 v10, 0x7fffffc0, v10 + global_store_dwordx2 v[23:24], v[30:31], off + s_cmp_eq_u32 s2, 0 + s_cbranch_scc1 main_loop_end + s_sub_i32 s2, s2, 1 + v_mov_b32 v37, v36 + v_mov_b32 v23, 0 + v_mov_b32 v36, v10 + v_mov_b32 v10, 0 + s_branch main_loop +main_loop_end: + + v_add_co_u32 v0, vcc, v3, v1 + v_addc_co_u32 v1, vcc, v4, 0, vcc + global_store_dwordx2 v[0:1], v[21:22], off + global_store_dwordx2 v[0:1], v[30:31], off inst_offset:64 + global_store_dwordx2 v[0:1], v[32:33], off inst_offset:128 + + # store rounding mode + v_mov_b32 v0, 0 + v_mov_b32 v1, s66 + global_store_dword v0, v1, s[64:65] + +program_end: + s_endpgm + +fsqrt_r_sub0: + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rsq_f64 v[28:29], v[68:69] + + # Improve initial approximation (can be skipped) + #v_mul_f64 v[42:43], v[28:29], v[68:69] + #v_mul_f64 v[48:49], v[28:29], -0.5 + #v_fma_f64 v[48:49], v[48:49], v[42:43], 0.5 + #v_fma_f64 v[28:29], v[28:29], v[48:49], v[28:29] + + v_mul_f64 v[42:43], v[28:29], v[68:69] + v_mov_b32 v48, v28 + v_sub_u32 v49, v29, v84 + v_mov_b32 v46, v28 + v_xor_b32 v47, v49, v82 + v_fma_f64 v[46:47], v[46:47], v[42:43], 0.5 + v_fma_f64 v[42:43], v[42:43], v[46:47], v[42:43] + v_fma_f64 v[48:49], v[48:49], v[46:47], v[48:49] + v_fma_f64 v[46:47], -v[42:43], v[42:43], v[68:69] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[46:47], v[48:49], v[42:43] + v_cmpx_class_f64 s[14:15], v[68:69], s[68:69] + v_mov_b32 v68, v42 + v_mov_b32 v69, v43 + s_mov_b64 exec, 3 + s_setpc_b64 s[60:61] + +fsqrt_r_sub1: + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rsq_f64 v[28:29], v[70:71] + + # Improve initial approximation (can be skipped) + #v_mul_f64 v[42:43], v[28:29], v[70:71] + #v_mul_f64 v[48:49], v[28:29], -0.5 + #v_fma_f64 v[48:49], v[48:49], v[42:43], 0.5 + #v_fma_f64 v[28:29], v[28:29], v[48:49], v[28:29] + + v_mul_f64 v[42:43], v[28:29], v[70:71] + v_mov_b32 v48, v28 + v_sub_u32 v49, v29, v84 + v_mov_b32 v46, v28 + v_xor_b32 v47, v49, v82 + v_fma_f64 v[46:47], v[46:47], v[42:43], 0.5 + v_fma_f64 v[42:43], v[42:43], v[46:47], v[42:43] + v_fma_f64 v[48:49], v[48:49], v[46:47], v[48:49] + v_fma_f64 v[46:47], -v[42:43], v[42:43], v[70:71] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[46:47], v[48:49], v[42:43] + v_cmpx_class_f64 s[14:15], v[70:71], s[68:69] + v_mov_b32 v70, v42 + v_mov_b32 v71, v43 + s_mov_b64 exec, 3 + s_setpc_b64 s[60:61] + +fsqrt_r_sub2: + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rsq_f64 v[28:29], v[72:73] + + # Improve initial approximation (can be skipped) + #v_mul_f64 v[42:43], v[28:29], v[72:73] + #v_mul_f64 v[48:49], v[28:29], -0.5 + #v_fma_f64 v[48:49], v[48:49], v[42:43], 0.5 + #v_fma_f64 v[28:29], v[28:29], v[48:49], v[28:29] + + v_mul_f64 v[42:43], v[28:29], v[72:73] + v_mov_b32 v48, v28 + v_sub_u32 v49, v29, v84 + v_mov_b32 v46, v28 + v_xor_b32 v47, v49, v82 + v_fma_f64 v[46:47], v[46:47], v[42:43], 0.5 + v_fma_f64 v[42:43], v[42:43], v[46:47], v[42:43] + v_fma_f64 v[48:49], v[48:49], v[46:47], v[48:49] + v_fma_f64 v[46:47], -v[42:43], v[42:43], v[72:73] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[46:47], v[48:49], v[42:43] + v_cmpx_class_f64 s[14:15], v[72:73], s[68:69] + v_mov_b32 v72, v42 + v_mov_b32 v73, v43 + s_mov_b64 exec, 3 + s_setpc_b64 s[60:61] + +fsqrt_r_sub3: + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rsq_f64 v[28:29], v[74:75] + + # Improve initial approximation (can be skipped) + #v_mul_f64 v[42:43], v[28:29], v[74:75] + #v_mul_f64 v[48:49], v[28:29], -0.5 + #v_fma_f64 v[48:49], v[48:49], v[42:43], 0.5 + #v_fma_f64 v[28:29], v[28:29], v[48:49], v[28:29] + + v_mul_f64 v[42:43], v[28:29], v[74:75] + v_mov_b32 v48, v28 + v_sub_u32 v49, v29, v84 + v_mov_b32 v46, v28 + v_xor_b32 v47, v49, v82 + v_fma_f64 v[46:47], v[46:47], v[42:43], 0.5 + v_fma_f64 v[42:43], v[42:43], v[46:47], v[42:43] + v_fma_f64 v[48:49], v[48:49], v[46:47], v[48:49] + v_fma_f64 v[46:47], -v[42:43], v[42:43], v[74:75] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[46:47], v[48:49], v[42:43] + v_cmpx_class_f64 s[14:15], v[74:75], s[68:69] + v_mov_b32 v74, v42 + v_mov_b32 v75, v43 + s_mov_b64 exec, 3 + s_setpc_b64 s[60:61] + +fdiv_m_sub0: + v_or_b32 v28, v28, v78 + v_and_or_b32 v29, v29, v77, v79 + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rcp_f64 v[48:49], v[28:29] + v_fma_f64 v[80:81], -v[28:29], v[48:49], 1.0 + v_fma_f64 v[48:49], v[48:49], v[80:81], v[48:49] + v_mul_f64 v[80:81], v[68:69], v[48:49] + v_fma_f64 v[42:43], -v[28:29], v[80:81], v[68:69] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[42:43], v[48:49], v[80:81] + v_div_fixup_f64 v[80:81], v[42:43], v[28:29], v[68:69] + v_cmpx_eq_f64 s[14:15], v[68:69], v[28:29] + v_mov_b32 v80, 0 + v_mov_b32 v81, v83 + s_mov_b64 exec, 3 + v_mov_b32 v68, v80 + v_mov_b32 v69, v81 + s_setpc_b64 s[60:61] + +fdiv_m_sub1: + v_or_b32 v28, v28, v78 + v_and_or_b32 v29, v29, v77, v79 + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rcp_f64 v[48:49], v[28:29] + v_fma_f64 v[80:81], -v[28:29], v[48:49], 1.0 + v_fma_f64 v[48:49], v[48:49], v[80:81], v[48:49] + v_mul_f64 v[80:81], v[70:71], v[48:49] + v_fma_f64 v[42:43], -v[28:29], v[80:81], v[70:71] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[42:43], v[48:49], v[80:81] + v_div_fixup_f64 v[80:81], v[42:43], v[28:29], v[70:71] + v_cmpx_eq_f64 s[14:15], v[70:71], v[28:29] + v_mov_b32 v80, 0 + v_mov_b32 v81, v83 + s_mov_b64 exec, 3 + v_mov_b32 v70, v80 + v_mov_b32 v71, v81 + s_setpc_b64 s[60:61] + +fdiv_m_sub2: + v_or_b32 v28, v28, v78 + v_and_or_b32 v29, v29, v77, v79 + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rcp_f64 v[48:49], v[28:29] + v_fma_f64 v[80:81], -v[28:29], v[48:49], 1.0 + v_fma_f64 v[48:49], v[48:49], v[80:81], v[48:49] + v_mul_f64 v[80:81], v[72:73], v[48:49] + v_fma_f64 v[42:43], -v[28:29], v[80:81], v[72:73] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[42:43], v[48:49], v[80:81] + v_div_fixup_f64 v[80:81], v[42:43], v[28:29], v[72:73] + v_cmpx_eq_f64 s[14:15], v[72:73], v[28:29] + v_mov_b32 v80, 0 + v_mov_b32 v81, v83 + s_mov_b64 exec, 3 + v_mov_b32 v72, v80 + v_mov_b32 v73, v81 + s_setpc_b64 s[60:61] + +fdiv_m_sub3: + v_or_b32 v28, v28, v78 + v_and_or_b32 v29, v29, v77, v79 + s_setreg_b32 hwreg(mode, 2, 2), s67 + v_rcp_f64 v[48:49], v[28:29] + v_fma_f64 v[80:81], -v[28:29], v[48:49], 1.0 + v_fma_f64 v[48:49], v[48:49], v[80:81], v[48:49] + v_mul_f64 v[80:81], v[74:75], v[48:49] + v_fma_f64 v[42:43], -v[28:29], v[80:81], v[74:75] + s_setreg_b32 hwreg(mode, 2, 2), s66 + v_fma_f64 v[42:43], v[42:43], v[48:49], v[80:81] + v_div_fixup_f64 v[80:81], v[42:43], v[28:29], v[74:75] + v_cmpx_eq_f64 s[14:15], v[74:75], v[28:29] + v_mov_b32 v80, 0 + v_mov_b32 v81, v83 + s_mov_b64 exec, 3 + v_mov_b32 v74, v80 + v_mov_b32 v75, v81 + s_setpc_b64 s[60:61] + +ismulh_r_sub: + s_mov_b64 exec, 1 + v_mov_b32 v45, s14 + v_mul_hi_u32 v40, s38, v45 + v_mov_b32 v47, s15 + v_mad_u64_u32 v[42:43], s[32:33], s38, v47, v[40:41] + v_mov_b32 v40, v42 + v_mad_u64_u32 v[45:46], s[32:33], s39, v45, v[40:41] + v_mad_u64_u32 v[42:43], s[32:33], s39, v47, v[43:44] + v_add_co_u32 v42, vcc, v42, v46 + v_addc_co_u32 v43, vcc, 0, v43, vcc + v_readlane_b32 s32, v42, 0 + v_readlane_b32 s33, v43, 0 + s_cmp_lt_i32 s15, 0 + s_cselect_b64 s[34:35], s[38:39], 0 + s_sub_u32 s32, s32, s34 + s_subb_u32 s33, s33, s35 + s_cmp_lt_i32 s39, 0 + s_cselect_b64 s[34:35], s[14:15], 0 + s_sub_u32 s14, s32, s34 + s_subb_u32 s15, s33, s35 + s_mov_b64 exec, 3 + s_setpc_b64 s[60:61] + +imulh_r_sub: + s_mov_b64 exec, 1 + v_mov_b32 v45, s38 + v_mul_hi_u32 v40, s14, v45 + v_mov_b32 v47, s39 + v_mad_u64_u32 v[42:43], s[32:33], s14, v47, v[40:41] + v_mov_b32 v40, v42 + v_mad_u64_u32 v[45:46], s[32:33], s15, v45, v[40:41] + v_mad_u64_u32 v[42:43], s[32:33], s15, v47, v[43:44] + v_add_co_u32 v42, vcc, v42, v46 + v_addc_co_u32 v43, vcc, 0, v43, vcc + v_readlane_b32 s14, v42, 0 + v_readlane_b32 s15, v43, 0 + s_mov_b64 exec, 3 + s_setpc_b64 s[60:61] diff --git a/src/backend/opencl/cl/rx/randomx_run_gfx900.h b/src/backend/opencl/cl/rx/randomx_run_gfx900.h new file mode 100644 index 00000000..977bb293 --- /dev/null +++ b/src/backend/opencl/cl/rx/randomx_run_gfx900.h @@ -0,0 +1,215 @@ +/* +This file was auto-generated from randomx_run_gfx900.asm: + +clrxasm randomx_run_gfx900.asm -o randomx_run_gfx900.bin +bin2h -c randomx_run_gfx900_bin < randomx_run_gfx900.bin > randomx_run_gfx900.h + +clrxasm can be downloaded here: https://github.com/CLRX/CLRX-mirror/releases +bin2h can be downloaded here: http://www.deadnode.org/sw/bin2h/ +*/ + +static unsigned char randomx_run_gfx900_bin[]={ +0x7f,0x45,0x4c,0x46,0x02,0x01,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x00,0x5b,0xaf,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xa0,0x17,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x07,0x00,0x01,0x00,0x00 +,0x2e,0x73,0x68,0x73,0x74,0x72,0x74,0x61,0x62,0x00,0x2e,0x73,0x74,0x72,0x74,0x61,0x62,0x00,0x2e,0x73,0x79,0x6d,0x74,0x61,0x62,0x00,0x2e,0x63,0x6f,0x6d,0x6d,0x65 +,0x6e,0x74,0x00,0x2e,0x72,0x6f,0x64,0x61,0x74,0x61,0x00,0x2e,0x74,0x65,0x78,0x74,0x00,0x00,0x5f,0x5f,0x4f,0x70,0x65,0x6e,0x43,0x4c,0x5f,0x26,0x5f,0x5f,0x4f,0x70 +,0x65,0x6e,0x43,0x4c,0x5f,0x72,0x61,0x6e,0x64,0x6f,0x6d,0x78,0x5f,0x72,0x75,0x6e,0x5f,0x6b,0x65,0x72,0x6e,0x65,0x6c,0x5f,0x6d,0x65,0x74,0x61,0x64,0x61,0x74,0x61 +,0x00,0x61,0x63,0x6c,0x5f,0x76,0x65,0x72,0x73,0x69,0x6f,0x6e,0x5f,0x73,0x74,0x72,0x69,0x6e,0x67,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x38 +,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x01,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x27,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41 +,0x4d,0x44,0x2d,0x43,0x4f,0x4d,0x50,0x2d,0x4c,0x49,0x42,0x2d,0x76,0x30,0x2e,0x38,0x20,0x28,0x30,0x2e,0x30,0x2e,0x53,0x43,0x5f,0x42,0x55,0x49,0x4c,0x44,0x5f,0x4e +,0x55,0x4d,0x42,0x45,0x52,0x29,0x10,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x38,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x68,0x00 +,0x00,0x00,0x24,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x40,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x5f,0x5f,0x4f,0x70,0x65,0x6e,0x43,0x4c,0x5f,0x64 +,0x75,0x6d,0x6d,0x79,0x5f,0x6b,0x65,0x72,0x6e,0x65,0x6c,0x00,0x47,0x46,0x58,0x39,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x05,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x0f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x01,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00 +,0x00,0x00,0x40,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x05,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00 +,0x00,0x00,0x04,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x03,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00 +,0x00,0x00,0x80,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x09,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x90,0x00,0x00,0x00,0x07,0x00 +,0x00,0x00,0x04,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xa0,0x00,0x00,0x00,0x07,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x08,0x00 +,0x00,0x00,0x04,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0xb0,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x0d,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00 +,0x00,0x00,0xc0,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x5f,0x2e,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x5f,0x6f,0x66,0x66,0x73,0x65,0x74,0x5f,0x30,0x00,0x73,0x69,0x7a,0x65,0x5f,0x74,0x00,0x5f,0x2e,0x67,0x6c,0x6f +,0x62,0x61,0x6c,0x5f,0x6f,0x66,0x66,0x73,0x65,0x74,0x5f,0x31,0x00,0x73,0x69,0x7a,0x65,0x5f,0x74,0x00,0x5f,0x2e,0x67,0x6c,0x6f,0x62,0x61,0x6c,0x5f,0x6f,0x66,0x66 +,0x73,0x65,0x74,0x5f,0x32,0x00,0x73,0x69,0x7a,0x65,0x5f,0x74,0x00,0x5f,0x2e,0x70,0x72,0x69,0x6e,0x74,0x66,0x5f,0x62,0x75,0x66,0x66,0x65,0x72,0x00,0x73,0x69,0x7a +,0x65,0x5f,0x74,0x00,0x5f,0x2e,0x76,0x71,0x75,0x65,0x75,0x65,0x5f,0x70,0x6f,0x69,0x6e,0x74,0x65,0x72,0x00,0x73,0x69,0x7a,0x65,0x5f,0x74,0x00,0x5f,0x2e,0x61,0x71 +,0x6c,0x77,0x72,0x61,0x70,0x5f,0x70,0x6f,0x69,0x6e,0x74,0x65,0x72,0x00,0x73,0x69,0x7a,0x65,0x5f,0x74,0x00,0x64,0x61,0x74,0x61,0x73,0x65,0x74,0x00,0x75,0x63,0x68 +,0x61,0x72,0x2a,0x00,0x73,0x63,0x72,0x61,0x74,0x63,0x68,0x70,0x61,0x64,0x00,0x75,0x63,0x68,0x61,0x72,0x2a,0x00,0x72,0x65,0x67,0x69,0x73,0x74,0x65,0x72,0x73,0x00 +,0x75,0x6c,0x6f,0x6e,0x67,0x2a,0x00,0x72,0x6f,0x75,0x6e,0x64,0x69,0x6e,0x67,0x5f,0x6d,0x6f,0x64,0x65,0x73,0x00,0x75,0x69,0x6e,0x74,0x2a,0x00,0x70,0x72,0x6f,0x67 +,0x72,0x61,0x6d,0x73,0x00,0x75,0x69,0x6e,0x74,0x2a,0x00,0x62,0x61,0x74,0x63,0x68,0x5f,0x73,0x69,0x7a,0x65,0x00,0x75,0x69,0x6e,0x74,0x00,0x72,0x78,0x5f,0x70,0x61 +,0x72,0x61,0x6d,0x65,0x74,0x65,0x72,0x73,0x00,0x75,0x69,0x6e,0x74,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7f,0x45 +,0x4c,0x46,0x02,0x01,0x01,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0xe0,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0xc0,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x38,0x00,0x01,0x00,0x40,0x00,0x06,0x00,0x01,0x00,0x03,0x00 +,0x00,0x60,0x05,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x74,0x0b +,0x00,0x00,0x00,0x00,0x00,0x00,0x74,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2e,0x73,0x68,0x73,0x74,0x72,0x74,0x61,0x62 +,0x00,0x2e,0x73,0x74,0x72,0x74,0x61,0x62,0x00,0x2e,0x6e,0x6f,0x74,0x65,0x00,0x2e,0x68,0x73,0x61,0x74,0x65,0x78,0x74,0x00,0x2e,0x73,0x79,0x6d,0x74,0x61,0x62,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x26,0x5f,0x5f,0x4f,0x70,0x65,0x6e,0x43,0x4c,0x5f,0x72,0x61,0x6e,0x64,0x6f,0x6d,0x78,0x5f,0x72,0x75,0x6e,0x5f,0x6b,0x65,0x72 +,0x6e,0x65,0x6c,0x00,0x5f,0x5f,0x68,0x73,0x61,0x5f,0x73,0x65,0x63,0x74,0x69,0x6f,0x6e,0x2e,0x68,0x73,0x61,0x74,0x65,0x78,0x74,0x00,0x00,0x00,0x00,0x00,0x04,0x00 +,0x00,0x00,0x08,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x41,0x4d,0x44,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x02,0x00 +,0x00,0x00,0x41,0x4d,0x44,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x01,0x01,0x00,0x04,0x00,0x00,0x00,0x1a,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x41,0x4d +,0x44,0x00,0x04,0x00,0x07,0x00,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x4d,0x44,0x00,0x41,0x4d,0x44,0x47,0x50,0x55,0x00,0x00,0x04,0x00 +,0x00,0x00,0x29,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x41,0x4d,0x44,0x00,0x19,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x41,0x4d,0x44,0x20,0x48,0x53 +,0x41,0x20,0x52,0x75,0x6e,0x74,0x69,0x6d,0x65,0x20,0x46,0x69,0x6e,0x61,0x6c,0x69,0x7a,0x65,0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x1a,0x00 +,0x00,0x00,0x05,0x00,0x00,0x00,0x41,0x4d,0x44,0x00,0x16,0x00,0x2d,0x68,0x73,0x61,0x5f,0x63,0x61,0x6c,0x6c,0x5f,0x63,0x6f,0x6e,0x76,0x65,0x6e,0x74,0x69,0x6f,0x6e +,0x3d,0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00 +,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x5f,0x03,0xac,0x00,0x90,0x00,0x00,0x00,0x29,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x01 +,0x00,0x00,0x00,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x62,0x00,0x80,0x00,0x80,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x04,0x04,0x04,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xff,0x00 +,0xfc,0xbe,0x00,0x00,0x01,0x00,0x00,0x00,0x84,0xc0,0x00,0x00,0x00,0x00,0x70,0x00,0x8c,0xbf,0x00,0x00,0x93,0xbf,0x00,0x00,0x82,0xbf,0x01,0x00,0xfd,0xd1,0x08,0x0c +,0x01,0x04,0x02,0x00,0x06,0xc0,0x00,0x00,0x00,0x00,0x82,0x00,0x06,0xc0,0x40,0x00,0x00,0x00,0x02,0x10,0x06,0xc0,0x48,0x00,0x00,0x00,0x7f,0xc0,0x8c,0xbf,0x08,0x82 +,0x10,0x8e,0x40,0x10,0x40,0x80,0x41,0x80,0x41,0x82,0x80,0x02,0x10,0x7e,0x00,0x80,0x50,0xdc,0x08,0x00,0x40,0x08,0x70,0x0f,0x8c,0xbf,0x42,0x00,0x89,0xd2,0x08,0x01 +,0x01,0x00,0x81,0x08,0x42,0xb9,0x80,0x00,0xc3,0xbe,0xff,0x00,0xc4,0xbe,0x00,0x01,0x00,0x00,0x80,0x00,0xc5,0xbe,0x00,0x02,0x02,0x68,0x86,0x02,0x04,0x20,0x85,0x04 +,0x06,0x24,0xbf,0x02,0x02,0x26,0x80,0x02,0x08,0x7e,0x03,0x00,0x8f,0xd2,0x83,0x06,0x02,0x00,0x84,0x02,0x0a,0x24,0x02,0x06,0x06,0x32,0x03,0x02,0x0c,0x7e,0x06,0x09 +,0x08,0x38,0x82,0x02,0x52,0x24,0x03,0x53,0x0c,0x32,0x07,0x6a,0x1c,0xd1,0x04,0x01,0xa9,0x01,0x00,0x80,0x50,0xdc,0x06,0x00,0x7f,0x06,0x80,0x02,0x00,0x7e,0x70,0x0f +,0x8c,0xbf,0x00,0x00,0x1a,0xd8,0x29,0x06,0x00,0x00,0x7f,0xc0,0x8c,0xbf,0x7e,0x01,0x80,0xbe,0x02,0x00,0xdb,0xd0,0x01,0x0f,0x01,0x00,0x68,0x01,0x88,0xbf,0x02,0x05 +,0x02,0xc0,0x5c,0x00,0x00,0x00,0x7f,0xc0,0x8c,0xbf,0x14,0xff,0x95,0x92,0x00,0x00,0x05,0x00,0x81,0x15,0x15,0x8e,0x14,0xff,0x96,0x92,0x05,0x00,0x05,0x00,0x81,0x16 +,0x16,0x8e,0x14,0xff,0x97,0x92,0x0a,0x00,0x05,0x00,0x81,0x17,0x17,0x8e,0x14,0xff,0x98,0x92,0x0f,0x00,0x04,0x00,0x81,0x18,0x18,0x8e,0x17,0xc0,0x02,0x80,0x14,0x00 +,0x86,0xd2,0x02,0x05,0x00,0x00,0x02,0x00,0x85,0xd2,0x02,0x05,0x00,0x00,0x80,0x02,0x52,0x7e,0x80,0x02,0x58,0x7e,0x98,0x00,0x6c,0xd8,0x00,0x00,0x00,0x06,0x02,0x00 +,0xc9,0xd0,0x01,0x09,0x01,0x00,0x12,0x10,0xee,0xd8,0x00,0x00,0x00,0x22,0x88,0x00,0xec,0xd8,0x00,0x00,0x00,0x0b,0x00,0x00,0x09,0xb0,0x7e,0x01,0x86,0xbe,0x06,0x02 +,0xfe,0x89,0xa0,0x00,0xec,0xd8,0x00,0x00,0x00,0x0d,0x06,0x7e,0xfe,0x89,0x80,0x02,0x1a,0x7e,0x80,0x02,0x1c,0x7e,0x06,0x01,0xfe,0xbe,0x08,0x01,0x86,0xbe,0x40,0x27 +,0x86,0xb7,0x05,0x00,0xff,0xd1,0x00,0x0b,0x02,0x03,0x7e,0x01,0x88,0xbe,0x08,0x02,0xfe,0x89,0xa8,0x00,0xec,0xd8,0x00,0x00,0x00,0x0f,0x08,0x7e,0xfe,0x89,0x80,0x02 +,0x1e,0x7e,0x80,0x02,0x20,0x7e,0x08,0x01,0xfe,0xbe,0x02,0x02,0x0a,0xc0,0x30,0x00,0x00,0x00,0x02,0x04,0x02,0xc0,0x58,0x00,0x00,0x00,0x02,0x01,0x06,0xc0,0x50,0x00 +,0x00,0x00,0x83,0x02,0x02,0x24,0x00,0x03,0x22,0x68,0x7f,0xc0,0x8c,0xbf,0x0a,0x04,0x04,0x32,0x0b,0x02,0x24,0x7e,0x12,0x29,0x24,0x38,0xff,0x02,0x26,0x7e,0xff,0xff +,0xff,0x00,0x08,0x0c,0x0c,0x32,0x09,0x02,0x28,0x7e,0x14,0x6a,0x1c,0xd1,0x14,0x01,0xa9,0x01,0x00,0x00,0xec,0xd8,0x11,0x00,0x00,0x15,0x04,0x06,0x04,0x80,0x05,0x07 +,0x05,0x82,0x13,0x00,0x00,0xd1,0x13,0x83,0x09,0x00,0x08,0x00,0xfd,0xd1,0x23,0x07,0x01,0x04,0x07,0x00,0xfd,0xd1,0x22,0x07,0x01,0x04,0x0c,0x00,0xfd,0xd1,0x0c,0x07 +,0x01,0x04,0x00,0x00,0xfd,0xd1,0x0b,0x07,0x01,0x04,0x24,0x03,0x14,0x7e,0x25,0x03,0x2e,0x7e,0x18,0x81,0x82,0x80,0x10,0x00,0x83,0xbe,0x26,0x00,0x35,0xd1,0x15,0x10 +,0x01,0x00,0x27,0x00,0x35,0xd1,0x16,0x10,0x01,0x00,0x32,0x00,0x35,0xd1,0x17,0x10,0x01,0x00,0xff,0x02,0x66,0x7e,0x00,0x00,0xf0,0x80,0x00,0x00,0x89,0xd2,0x02,0x01 +,0x01,0x00,0x01,0x00,0x89,0xd2,0x12,0x01,0x01,0x00,0x7e,0x01,0xa4,0xbe,0x82,0x01,0xfe,0xbe,0x88,0x02,0x52,0x7e,0x84,0x02,0x58,0x7e,0x83,0x01,0xfe,0xbe,0x18,0x1a +,0xee,0xd8,0x29,0x00,0x00,0x34,0x1c,0x1e,0xee,0xd8,0x29,0x00,0x00,0x38,0xff,0x02,0x9a,0x7e,0xff,0xff,0xff,0x00,0xa0,0x00,0xec,0xd8,0x29,0x00,0x00,0x4e,0x24,0x01 +,0xfe,0xbe,0xff,0x02,0xa4,0x7e,0x00,0x00,0x00,0x80,0xff,0x02,0xa6,0x7e,0x00,0x00,0xf0,0x3f,0xff,0x02,0xa8,0x7e,0x00,0x00,0x10,0x00,0x00,0x1c,0x8e,0xbe,0x0e,0xff +,0x28,0x80,0xc4,0x03,0x00,0x00,0x0f,0x80,0x29,0x82,0x0e,0xff,0x2a,0x80,0x28,0x04,0x00,0x00,0x0f,0x80,0x2b,0x82,0x0e,0xff,0x2c,0x80,0x8c,0x04,0x00,0x00,0x0f,0x80 +,0x2d,0x82,0x0e,0xff,0x2e,0x80,0xf0,0x04,0x00,0x00,0x0f,0x80,0x2f,0x82,0x0e,0xff,0x30,0x80,0x54,0x05,0x00,0x00,0x0f,0x80,0x31,0x82,0x0e,0xff,0x32,0x80,0xbc,0x05 +,0x00,0x00,0x0f,0x80,0x33,0x82,0x0e,0xff,0x34,0x80,0x24,0x06,0x00,0x00,0x0f,0x80,0x35,0x82,0x0e,0xff,0x36,0x80,0x8c,0x06,0x00,0x00,0x0f,0x80,0x37,0x82,0x0e,0xff +,0x38,0x80,0xf4,0x06,0x00,0x00,0x0f,0x80,0x39,0x82,0x0e,0xff,0x3a,0x80,0x64,0x07,0x00,0x00,0x0f,0x80,0x3b,0x82,0xc1,0x00,0xbf,0xbe,0xff,0x00,0xc6,0xbe,0x00,0xff +,0x00,0x00,0xff,0x00,0xc7,0xbe,0x00,0xfe,0x01,0x00,0xff,0x00,0xc8,0xbe,0x00,0xfc,0x03,0x00,0xff,0x00,0xc9,0xbe,0x00,0xf8,0x07,0x00,0xff,0x00,0xca,0xbe,0x00,0xf0 +,0x0f,0x00,0xff,0x00,0xcb,0xbe,0x00,0xe0,0x1f,0x00,0xff,0x00,0xcc,0xbe,0x00,0xc0,0x3f,0x00,0xff,0x00,0xcd,0xbe,0x00,0x80,0x7f,0x00,0xff,0x00,0xce,0xbe,0x00,0x00 +,0xff,0x00,0xff,0x00,0xcf,0xbe,0x00,0x00,0xfe,0x01,0xff,0x00,0xd0,0xbe,0x00,0x00,0xfc,0x03,0xff,0x00,0xd1,0xbe,0x00,0x00,0xf8,0x07,0xff,0x00,0xd2,0xbe,0x00,0x00 +,0xf0,0x0f,0xff,0x00,0xd3,0xbe,0x00,0x00,0xe0,0x1f,0xff,0x00,0xd4,0xbe,0x00,0x00,0xc0,0x3f,0xff,0x00,0xd5,0xbe,0x00,0x00,0x80,0x7f,0x17,0xc0,0xd6,0x80,0x00,0x00 +,0xec,0xd8,0x00,0x00,0x00,0x18,0x00,0x00,0xec,0xd8,0x0c,0x00,0x00,0x1a,0x7f,0xc0,0x8c,0xbf,0x1b,0x33,0x32,0x2a,0x1a,0x31,0x30,0x2a,0x19,0x15,0x14,0x2a,0x18,0x2f +,0x2e,0x2a,0x56,0x14,0x14,0x26,0x56,0x2e,0x2e,0x26,0x0a,0x03,0x14,0x68,0x17,0x03,0x2e,0x68,0x02,0x15,0x34,0x32,0x1b,0x6a,0x1c,0xd1,0x12,0x01,0xa9,0x01,0x02,0x2f +,0x2e,0x32,0x18,0x6a,0x1c,0xd1,0x12,0x01,0xa9,0x01,0x00,0x80,0x54,0xdc,0x1a,0x00,0x7f,0x1c,0x00,0x80,0x54,0xdc,0x17,0x00,0x7f,0x1e,0x71,0x0f,0x8c,0xbf,0x1c,0x09 +,0x40,0x7e,0x1d,0x09,0x38,0x7e,0x70,0x0f,0x8c,0xbf,0x15,0x3d,0x44,0x2a,0x16,0x3f,0x46,0x2a,0x06,0x49,0x2c,0x32,0x19,0x6a,0x1c,0xd1,0x14,0x01,0xa9,0x01,0x16,0x03 +,0x2a,0x32,0x16,0x6a,0x1c,0xd1,0x19,0x01,0xa9,0x01,0x00,0x80,0x54,0xdc,0x15,0x00,0x7f,0x15,0x20,0x1b,0x3c,0x28,0x1f,0x00,0x01,0xd2,0x21,0x27,0x3a,0x04,0x1c,0x1f +,0x38,0x28,0x1d,0x00,0x01,0xd2,0x1d,0x27,0x42,0x04,0x00,0x01,0x9c,0xd8,0x05,0x1e,0x1c,0x00,0x7f,0xc0,0x8c,0xbf,0x83,0x01,0xfe,0xbe,0x08,0x0a,0xee,0xd8,0x29,0x00 +,0x00,0x3c,0x0c,0x0e,0xee,0xd8,0x29,0x00,0x00,0x40,0x10,0x12,0xee,0xd8,0x29,0x00,0x00,0x44,0x14,0x16,0xee,0xd8,0x29,0x00,0x00,0x48,0x10,0x00,0x89,0xd2,0x22,0x01 +,0x01,0x00,0x11,0x00,0x89,0xd2,0x23,0x01,0x01,0x00,0x12,0x00,0x89,0xd2,0x22,0x03,0x01,0x00,0x13,0x00,0x89,0xd2,0x23,0x03,0x01,0x00,0x14,0x00,0x89,0xd2,0x22,0x05 +,0x01,0x00,0x15,0x00,0x89,0xd2,0x23,0x05,0x01,0x00,0x16,0x00,0x89,0xd2,0x22,0x07,0x01,0x00,0x17,0x00,0x89,0xd2,0x23,0x07,0x01,0x00,0x18,0x00,0x89,0xd2,0x22,0x09 +,0x01,0x00,0x19,0x00,0x89,0xd2,0x23,0x09,0x01,0x00,0x1a,0x00,0x89,0xd2,0x22,0x0b,0x01,0x00,0x1b,0x00,0x89,0xd2,0x23,0x0b,0x01,0x00,0x1c,0x00,0x89,0xd2,0x22,0x0d +,0x01,0x00,0x1d,0x00,0x89,0xd2,0x23,0x0d,0x01,0x00,0x1e,0x00,0x89,0xd2,0x22,0x0f,0x01,0x00,0x1f,0x00,0x89,0xd2,0x23,0x0f,0x01,0x00,0x7f,0xc0,0x8c,0xbf,0x04,0x1e +,0x8c,0xbe,0x08,0x0a,0x9c,0xd8,0x29,0x3c,0x3e,0x00,0x0c,0x0e,0x9c,0xd8,0x29,0x40,0x42,0x00,0x10,0x12,0x9c,0xd8,0x29,0x44,0x46,0x00,0x14,0x16,0x9c,0xd8,0x29,0x48 +,0x4a,0x00,0x1c,0x00,0x8a,0xd2,0x10,0x00,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x11,0x00,0x01,0x00,0x1c,0x00,0x8a,0xd2,0x12,0x02,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x13,0x02 +,0x01,0x00,0x1c,0x00,0x8a,0xd2,0x14,0x04,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x15,0x04,0x01,0x00,0x1c,0x00,0x8a,0xd2,0x16,0x06,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x17,0x06 +,0x01,0x00,0x1c,0x00,0x8a,0xd2,0x18,0x08,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x19,0x08,0x01,0x00,0x1c,0x00,0x8a,0xd2,0x1a,0x0a,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x1b,0x0a +,0x01,0x00,0x1c,0x00,0x8a,0xd2,0x1c,0x0c,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x1d,0x0c,0x01,0x00,0x1c,0x00,0x8a,0xd2,0x1e,0x0e,0x01,0x00,0x1d,0x00,0x8a,0xd2,0x1f,0x0e +,0x01,0x00,0x24,0x01,0xfe,0xbe,0x00,0x00,0x9a,0xd8,0x11,0x1c,0x00,0x00,0x7f,0xc0,0x8c,0xbf,0x1c,0x2b,0x2a,0x2a,0x1d,0x2d,0x2c,0x2a,0x00,0x00,0x6c,0xd8,0x07,0x00 +,0x00,0x1c,0x00,0x00,0x6c,0xd8,0x08,0x00,0x00,0x1d,0x00,0x00,0x9a,0xd8,0x11,0x15,0x00,0x00,0x7f,0xc1,0x8c,0xbf,0x08,0x10,0xee,0xd8,0x11,0x00,0x00,0x1e,0x1c,0x4b +,0x14,0x2a,0x7f,0xc0,0x8c,0xbf,0x20,0x3d,0x3c,0x2a,0x21,0x3f,0x3e,0x2a,0x0a,0x3b,0x14,0x2a,0x00,0x80,0x74,0xdc,0x1a,0x15,0x7f,0x00,0xff,0x14,0x14,0x26,0xc0,0xff +,0xff,0x7f,0x00,0x80,0x74,0xdc,0x17,0x1e,0x7f,0x00,0x02,0x80,0x06,0xbf,0x06,0x00,0x85,0xbf,0x02,0x81,0x82,0x81,0x24,0x03,0x4a,0x7e,0x80,0x02,0x2e,0x7e,0x0a,0x03 +,0x48,0x7e,0x80,0x02,0x14,0x7e,0x5d,0xff,0x82,0xbf,0x03,0x03,0x00,0x32,0x01,0x6a,0x1c,0xd1,0x04,0x01,0xa9,0x01,0x00,0x80,0x74,0xdc,0x00,0x15,0x7f,0x00,0x40,0x80 +,0x74,0xdc,0x00,0x1e,0x7f,0x00,0x80,0x80,0x74,0xdc,0x00,0x20,0x7f,0x00,0x80,0x02,0x00,0x7e,0x42,0x02,0x02,0x7e,0x00,0x80,0x70,0xdc,0x00,0x01,0x40,0x00,0x00,0x00 +,0x81,0xbf,0x81,0x08,0x43,0xb9,0x44,0x4d,0x38,0x7e,0x2a,0x00,0x81,0xd2,0x1c,0x89,0x02,0x00,0x1c,0x03,0x60,0x7e,0x1d,0xa9,0x62,0x6a,0x1c,0x03,0x5c,0x7e,0x31,0xa5 +,0x5e,0x2a,0x2e,0x00,0xcc,0xd1,0x2e,0x55,0xc2,0x03,0x2a,0x00,0xcc,0xd1,0x2a,0x5d,0xaa,0x04,0x30,0x00,0xcc,0xd1,0x30,0x5d,0xc2,0x04,0x2e,0x00,0xcc,0xd1,0x2a,0x55 +,0x12,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2e,0x61,0xaa,0x04,0x0e,0x00,0x13,0xd0,0x44,0x89,0x00,0x00,0x2a,0x03,0x88,0x7e,0x2b,0x03,0x8a,0x7e,0x83,0x01 +,0xfe,0xbe,0x3c,0x1d,0x80,0xbe,0x81,0x08,0x43,0xb9,0x46,0x4d,0x38,0x7e,0x2a,0x00,0x81,0xd2,0x1c,0x8d,0x02,0x00,0x1c,0x03,0x60,0x7e,0x1d,0xa9,0x62,0x6a,0x1c,0x03 +,0x5c,0x7e,0x31,0xa5,0x5e,0x2a,0x2e,0x00,0xcc,0xd1,0x2e,0x55,0xc2,0x03,0x2a,0x00,0xcc,0xd1,0x2a,0x5d,0xaa,0x04,0x30,0x00,0xcc,0xd1,0x30,0x5d,0xc2,0x04,0x2e,0x00 +,0xcc,0xd1,0x2a,0x55,0x1a,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2e,0x61,0xaa,0x04,0x0e,0x00,0x13,0xd0,0x46,0x89,0x00,0x00,0x2a,0x03,0x8c,0x7e,0x2b,0x03 +,0x8e,0x7e,0x83,0x01,0xfe,0xbe,0x3c,0x1d,0x80,0xbe,0x81,0x08,0x43,0xb9,0x48,0x4d,0x38,0x7e,0x2a,0x00,0x81,0xd2,0x1c,0x91,0x02,0x00,0x1c,0x03,0x60,0x7e,0x1d,0xa9 +,0x62,0x6a,0x1c,0x03,0x5c,0x7e,0x31,0xa5,0x5e,0x2a,0x2e,0x00,0xcc,0xd1,0x2e,0x55,0xc2,0x03,0x2a,0x00,0xcc,0xd1,0x2a,0x5d,0xaa,0x04,0x30,0x00,0xcc,0xd1,0x30,0x5d +,0xc2,0x04,0x2e,0x00,0xcc,0xd1,0x2a,0x55,0x22,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2e,0x61,0xaa,0x04,0x0e,0x00,0x13,0xd0,0x48,0x89,0x00,0x00,0x2a,0x03 +,0x90,0x7e,0x2b,0x03,0x92,0x7e,0x83,0x01,0xfe,0xbe,0x3c,0x1d,0x80,0xbe,0x81,0x08,0x43,0xb9,0x4a,0x4d,0x38,0x7e,0x2a,0x00,0x81,0xd2,0x1c,0x95,0x02,0x00,0x1c,0x03 +,0x60,0x7e,0x1d,0xa9,0x62,0x6a,0x1c,0x03,0x5c,0x7e,0x31,0xa5,0x5e,0x2a,0x2e,0x00,0xcc,0xd1,0x2e,0x55,0xc2,0x03,0x2a,0x00,0xcc,0xd1,0x2a,0x5d,0xaa,0x04,0x30,0x00 +,0xcc,0xd1,0x30,0x5d,0xc2,0x04,0x2e,0x00,0xcc,0xd1,0x2a,0x55,0x2a,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2e,0x61,0xaa,0x04,0x0e,0x00,0x13,0xd0,0x4a,0x89 +,0x00,0x00,0x2a,0x03,0x94,0x7e,0x2b,0x03,0x96,0x7e,0x83,0x01,0xfe,0xbe,0x3c,0x1d,0x80,0xbe,0x1c,0x9d,0x38,0x28,0x1d,0x00,0x01,0xd2,0x1d,0x9b,0x3e,0x05,0x81,0x08 +,0x43,0xb9,0x1c,0x4b,0x60,0x7e,0x50,0x00,0xcc,0xd1,0x1c,0x61,0xca,0x23,0x30,0x00,0xcc,0xd1,0x30,0xa1,0xc2,0x04,0x50,0x00,0x81,0xd2,0x44,0x61,0x02,0x00,0x2a,0x00 +,0xcc,0xd1,0x1c,0xa1,0x12,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2a,0x61,0x42,0x05,0x50,0x00,0xdf,0xd1,0x2a,0x39,0x12,0x05,0x0e,0x00,0x72,0xd0,0x44,0x39 +,0x02,0x00,0x80,0x02,0xa0,0x7e,0x53,0x03,0xa2,0x7e,0x83,0x01,0xfe,0xbe,0x50,0x03,0x88,0x7e,0x51,0x03,0x8a,0x7e,0x3c,0x1d,0x80,0xbe,0x1c,0x9d,0x38,0x28,0x1d,0x00 +,0x01,0xd2,0x1d,0x9b,0x3e,0x05,0x81,0x08,0x43,0xb9,0x1c,0x4b,0x60,0x7e,0x50,0x00,0xcc,0xd1,0x1c,0x61,0xca,0x23,0x30,0x00,0xcc,0xd1,0x30,0xa1,0xc2,0x04,0x50,0x00 +,0x81,0xd2,0x46,0x61,0x02,0x00,0x2a,0x00,0xcc,0xd1,0x1c,0xa1,0x1a,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2a,0x61,0x42,0x05,0x50,0x00,0xdf,0xd1,0x2a,0x39 +,0x1a,0x05,0x0e,0x00,0x72,0xd0,0x46,0x39,0x02,0x00,0x80,0x02,0xa0,0x7e,0x53,0x03,0xa2,0x7e,0x83,0x01,0xfe,0xbe,0x50,0x03,0x8c,0x7e,0x51,0x03,0x8e,0x7e,0x3c,0x1d +,0x80,0xbe,0x1c,0x9d,0x38,0x28,0x1d,0x00,0x01,0xd2,0x1d,0x9b,0x3e,0x05,0x81,0x08,0x43,0xb9,0x1c,0x4b,0x60,0x7e,0x50,0x00,0xcc,0xd1,0x1c,0x61,0xca,0x23,0x30,0x00 +,0xcc,0xd1,0x30,0xa1,0xc2,0x04,0x50,0x00,0x81,0xd2,0x48,0x61,0x02,0x00,0x2a,0x00,0xcc,0xd1,0x1c,0xa1,0x22,0x25,0x81,0x08,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2a,0x61 +,0x42,0x05,0x50,0x00,0xdf,0xd1,0x2a,0x39,0x22,0x05,0x0e,0x00,0x72,0xd0,0x48,0x39,0x02,0x00,0x80,0x02,0xa0,0x7e,0x53,0x03,0xa2,0x7e,0x83,0x01,0xfe,0xbe,0x50,0x03 +,0x90,0x7e,0x51,0x03,0x92,0x7e,0x3c,0x1d,0x80,0xbe,0x1c,0x9d,0x38,0x28,0x1d,0x00,0x01,0xd2,0x1d,0x9b,0x3e,0x05,0x81,0x08,0x43,0xb9,0x1c,0x4b,0x60,0x7e,0x50,0x00 +,0xcc,0xd1,0x1c,0x61,0xca,0x23,0x30,0x00,0xcc,0xd1,0x30,0xa1,0xc2,0x04,0x50,0x00,0x81,0xd2,0x4a,0x61,0x02,0x00,0x2a,0x00,0xcc,0xd1,0x1c,0xa1,0x2a,0x25,0x81,0x08 +,0x42,0xb9,0x2a,0x00,0xcc,0xd1,0x2a,0x61,0x42,0x05,0x50,0x00,0xdf,0xd1,0x2a,0x39,0x2a,0x05,0x0e,0x00,0x72,0xd0,0x4a,0x39,0x02,0x00,0x80,0x02,0xa0,0x7e,0x53,0x03 +,0xa2,0x7e,0x83,0x01,0xfe,0xbe,0x50,0x03,0x94,0x7e,0x51,0x03,0x96,0x7e,0x3c,0x1d,0x80,0xbe,0x81,0x01,0xfe,0xbe,0x0e,0x02,0x5a,0x7e,0x28,0x00,0x86,0xd2,0x26,0x5a +,0x02,0x00,0x0f,0x02,0x5e,0x7e,0x2a,0x20,0xe8,0xd1,0x26,0x5e,0xa2,0x04,0x2a,0x03,0x50,0x7e,0x2d,0x20,0xe8,0xd1,0x27,0x5a,0xa2,0x04,0x2a,0x20,0xe8,0xd1,0x27,0x5e +,0xae,0x04,0x2a,0x5d,0x54,0x32,0x80,0x56,0x56,0x38,0x20,0x00,0x89,0xd2,0x2a,0x01,0x01,0x00,0x21,0x00,0x89,0xd2,0x2b,0x01,0x01,0x00,0x0f,0x80,0x04,0xbf,0x26,0x80 +,0xa2,0x85,0x20,0x22,0xa0,0x80,0x21,0x23,0xa1,0x82,0x27,0x80,0x04,0xbf,0x0e,0x80,0xa2,0x85,0x20,0x22,0x8e,0x80,0x21,0x23,0x8f,0x82,0x83,0x01,0xfe,0xbe,0x3c,0x1d +,0x80,0xbe,0x81,0x01,0xfe,0xbe,0x26,0x02,0x5a,0x7e,0x28,0x00,0x86,0xd2,0x0e,0x5a,0x02,0x00,0x27,0x02,0x5e,0x7e,0x2a,0x20,0xe8,0xd1,0x0e,0x5e,0xa2,0x04,0x2a,0x03 +,0x50,0x7e,0x2d,0x20,0xe8,0xd1,0x0f,0x5a,0xa2,0x04,0x2a,0x20,0xe8,0xd1,0x0f,0x5e,0xae,0x04,0x2a,0x5d,0x54,0x32,0x80,0x56,0x56,0x38,0x0e,0x00,0x89,0xd2,0x2a,0x01 +,0x01,0x00,0x0f,0x00,0x89,0xd2,0x2b,0x01,0x01,0x00,0x83,0x01,0xfe,0xbe,0x3c,0x1d,0x80,0xbe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x1a,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x74,0x0b +,0x00,0x00,0x00,0x00,0x00,0x00,0x1e,0x00,0x00,0x00,0x03,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00 +,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x78,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2a,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x0b,0x00 +,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xa8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x34,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x13,0x00 +,0x00,0x00,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xe0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xc8,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x19,0x00 +,0x00,0x00,0x01,0x00,0x00,0x00,0x07,0x00,0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x74,0x0b +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x22,0x00 +,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x78,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01 +,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x32 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x0b +,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x43 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x13 +,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xb8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x18,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1b +,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x27 +,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x24 +,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x27,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x38 +,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2c +,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x5f,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x40 +,0x0f,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; +const int randomx_run_gfx900_bin_size=6496; diff --git a/src/backend/opencl/cl/rx/randomx_vm.cl b/src/backend/opencl/cl/rx/randomx_vm.cl new file mode 100644 index 00000000..7204467a --- /dev/null +++ b/src/backend/opencl/cl/rx/randomx_vm.cl @@ -0,0 +1,2070 @@ +R"===( +/* +Copyright (c) 2019 SChernykh +Portions Copyright (c) 2018-2019 tevador + +This file is part of RandomX OpenCL. + +RandomX OpenCL is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +RandomX OpenCL is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with RandomX OpenCL. If not, see . +*/ + +#pragma OPENCL EXTENSION cl_khr_fp64 : enable + +#define CacheLineSize 64 +#define ScratchpadL3Mask64 (RANDOMX_SCRATCHPAD_L3 - CacheLineSize) +#define CacheLineAlignMask ((RANDOMX_DATASET_BASE_SIZE - 1) & ~(CacheLineSize - 1)) + +#define mantissaSize 52 +#define exponentSize 11 +#define mantissaMask ((1UL << mantissaSize) - 1) +#define exponentMask ((1UL << exponentSize) - 1) +#define exponentBias 1023 +#define constExponentBits 0x300 +#define dynamicExponentBits 4 +#define staticExponentBits 4 +#define dynamicMantissaMask ((1UL << (mantissaSize + dynamicExponentBits)) - 1) + +#define RegistersCount 8 +#define RegisterCountFlt (RegistersCount / 2) +#define ConditionMask ((1 << RANDOMX_JUMP_BITS) - 1) +#define ConditionOffset RANDOMX_JUMP_OFFSET +#define StoreL3Condition 14 +#define DatasetExtraItems (RANDOMX_DATASET_EXTRA_SIZE / RANDOMX_DATASET_ITEM_SIZE) + +#define RegisterNeedsDisplacement 5 + +// +// VM state: +// +// Bytes 0-255: registers +// Bytes 256-1023: imm32 values (up to 192 values can be stored). IMUL_RCP and CBRANCH use 2 consecutive imm32 values. +// Bytes 1024-2047: up to 256 instructions +// +// Instruction encoding: +// +// Bits 0-2: dst (0-7) +// Bits 3-5: src (0-7) +// Bits 6-13: imm32/64 offset (in DWORDs, 0-191) +// Bit 14: src location (register, scratchpad) +// Bits 15-16: src shift (0-3), ADD/MUL switch for FMA instruction +// Bit 17: src=imm32 +// Bit 18: src=imm64 +// Bit 19: src = -src +// Bits 20-23: opcode (add_rs, add, mul, umul_hi, imul_hi, neg, xor, ror, swap, cbranch, store, fswap, fma, fsqrt, fdiv, cfround) +// Bits 24-27: how many parallel instructions to run starting with this one (1-16) +// Bits 28-31: how many of them are FP instructions (0-8) +// + +#define DST_OFFSET 0 +#define SRC_OFFSET 3 +#define IMM_OFFSET 6 +#define LOC_OFFSET 14 +#define SHIFT_OFFSET 15 +#define SRC_IS_IMM32_OFFSET 17 +#define SRC_IS_IMM64_OFFSET 18 +#define NEGATIVE_SRC_OFFSET 19 +#define OPCODE_OFFSET 20 +#define NUM_INSTS_OFFSET 24 +#define NUM_FP_INSTS_OFFSET 28 + +// ISWAP r0, r0 +#define INST_NOP (8 << OPCODE_OFFSET) + +typedef uchar uint8_t; +typedef ushort uint16_t; +typedef uint uint32_t; +typedef ulong uint64_t; + +typedef int int32_t; +typedef long int64_t; + +double getSmallPositiveFloatBits(uint64_t entropy) +{ + uint64_t exponent = entropy >> 59; //0..31 + uint64_t mantissa = entropy & mantissaMask; + exponent += exponentBias; + exponent &= exponentMask; + exponent <<= mantissaSize; + return as_double(exponent | mantissa); +} + +uint64_t getStaticExponent(uint64_t entropy) +{ + uint64_t exponent = constExponentBits; + exponent |= (entropy >> (64 - staticExponentBits)) << dynamicExponentBits; + exponent <<= mantissaSize; + return exponent; +} + +uint64_t getFloatMask(uint64_t entropy) +{ + const uint64_t mask22bit = (1UL << 22) - 1; + return (entropy & mask22bit) | getStaticExponent(entropy); +} + +void set_buffer(__local uint32_t *dst_buf, uint32_t N, const uint32_t value) +{ + uint32_t i = get_local_id(0) * sizeof(uint32_t); + const uint32_t step = get_local_size(0) * sizeof(uint32_t); + __local uint8_t* dst = ((__local uint8_t*)dst_buf) + i; + while (i < sizeof(uint32_t) * N) + { + *(__local uint32_t*)(dst) = value; + dst += step; + i += step; + } +} + +uint64_t imul_rcp_value(uint32_t divisor) +{ + if ((divisor & (divisor - 1)) == 0) + { + return 1UL; + } + + const uint64_t p2exp63 = 1UL << 63; + + uint64_t quotient = p2exp63 / divisor; + uint64_t remainder = p2exp63 % divisor; + + const uint32_t bsr = 31 - clz(divisor); + + for (uint32_t shift = 0; shift <= bsr; ++shift) + { + const bool b = (remainder >= divisor - remainder); + quotient = (quotient << 1) | (b ? 1 : 0); + remainder = (remainder << 1) - (b ? divisor : 0); + } + + return quotient; +} + +)===" +R"===( + +#define set_byte(a, position, value) do { ((uint8_t*)&(a))[(position)] = (value); } while (0) +uint32_t get_byte(uint64_t a, uint32_t position) { return (a >> (position << 3)) & 0xFF; } +#define update_max(value, next_value) do { if ((value) < (next_value)) (value) = (next_value); } while (0) + +__attribute__((reqd_work_group_size(32, 1, 1))) +__kernel void init_vm(__global const void* entropy_data, __global void* vm_states, __global uint32_t* rounding, uint32_t iteration) +{ +#if RANDOMX_PROGRAM_SIZE <= 256 + typedef uint8_t exec_t; +#else + typedef uint16_t exec_t; +#endif + + __local uint32_t execution_plan_buf[RANDOMX_PROGRAM_SIZE * WORKERS_PER_HASH * (32 / 8) * sizeof(exec_t) / sizeof(uint32_t)]; + + set_buffer(execution_plan_buf, sizeof(execution_plan_buf) / sizeof(uint32_t), 0); + barrier(CLK_LOCAL_MEM_FENCE); + + const uint32_t global_index = get_global_id(0); + const uint32_t idx = global_index / 8; + const uint32_t sub = global_index % 8; + + __local exec_t* execution_plan = (__local exec_t*)(execution_plan_buf + (get_local_id(0) / 8) * RANDOMX_PROGRAM_SIZE * WORKERS_PER_HASH * sizeof(exec_t) / sizeof(uint32_t)); + + __global uint64_t* R = ((__global uint64_t*)vm_states) + idx * VM_STATE_SIZE / sizeof(uint64_t); + R[sub] = 0; + + const __global uint64_t* entropy = ((const __global uint64_t*)entropy_data) + idx * ENTROPY_SIZE / sizeof(uint64_t); + + __global double* A = (__global double*)(R + 24); + A[sub] = getSmallPositiveFloatBits(entropy[sub]); + + if (sub == 0) + { + if (iteration == 0) + rounding[idx] = 0; + + __global uint2* src_program = (__global uint2*)(entropy + 128 / sizeof(uint64_t)); + +#if RANDOMX_PROGRAM_SIZE <= 256 + uint64_t registerLastChanged = 0; + uint64_t registerWasChanged = 0; +#else + int32_t registerLastChanged[8] = { -1, -1, -1, -1, -1, -1, -1, -1 }; +#endif + + // Initialize CBRANCH instructions + for (uint32_t i = 0; i < RANDOMX_PROGRAM_SIZE; ++i) + { + // Clear all src flags (branch target, FP, branch) + *(__global uint32_t*)(src_program + i) &= ~(0xF8U << 8); + + const uint2 src_inst = src_program[i]; + uint2 inst = src_inst; + + uint32_t opcode = inst.x & 0xff; + const uint32_t dst = (inst.x >> 8) & 7; + const uint32_t src = (inst.x >> 16) & 7; + + if (opcode < RANDOMX_FREQ_IADD_RS + RANDOMX_FREQ_IADD_M + RANDOMX_FREQ_ISUB_R + RANDOMX_FREQ_ISUB_M + RANDOMX_FREQ_IMUL_R + RANDOMX_FREQ_IMUL_M + RANDOMX_FREQ_IMULH_R + RANDOMX_FREQ_IMULH_M + RANDOMX_FREQ_ISMULH_R + RANDOMX_FREQ_ISMULH_M) + { +#if RANDOMX_PROGRAM_SIZE <= 256 + set_byte(registerLastChanged, dst, i); + set_byte(registerWasChanged, dst, 1); +#else + registerLastChanged[dst] = i; +#endif + continue; + } + opcode -= RANDOMX_FREQ_IADD_RS + RANDOMX_FREQ_IADD_M + RANDOMX_FREQ_ISUB_R + RANDOMX_FREQ_ISUB_M + RANDOMX_FREQ_IMUL_R + RANDOMX_FREQ_IMUL_M + RANDOMX_FREQ_IMULH_R + RANDOMX_FREQ_IMULH_M + RANDOMX_FREQ_ISMULH_R + RANDOMX_FREQ_ISMULH_M; + + if (opcode < RANDOMX_FREQ_IMUL_RCP) + { + if (inst.y & (inst.y - 1)) + { +#if RANDOMX_PROGRAM_SIZE <= 256 + set_byte(registerLastChanged, dst, i); + set_byte(registerWasChanged, dst, 1); +#else + registerLastChanged[dst] = i; +#endif + } + continue; + } + opcode -= RANDOMX_FREQ_IMUL_RCP; + + if (opcode < RANDOMX_FREQ_INEG_R + RANDOMX_FREQ_IXOR_R + RANDOMX_FREQ_IXOR_M + RANDOMX_FREQ_IROR_R + RANDOMX_FREQ_IROL_R) + { +#if RANDOMX_PROGRAM_SIZE <= 256 + set_byte(registerLastChanged, dst, i); + set_byte(registerWasChanged, dst, 1); +#else + registerLastChanged[dst] = i; +#endif + continue; + } + opcode -= RANDOMX_FREQ_INEG_R + RANDOMX_FREQ_IXOR_R + RANDOMX_FREQ_IXOR_M + RANDOMX_FREQ_IROR_R + RANDOMX_FREQ_IROL_R; + + if (opcode < RANDOMX_FREQ_ISWAP_R) + { + if (src != dst) + { +#if RANDOMX_PROGRAM_SIZE <= 256 + set_byte(registerLastChanged, dst, i); + set_byte(registerWasChanged, dst, 1); + set_byte(registerLastChanged, src, i); + set_byte(registerWasChanged, src, 1); +#else + registerLastChanged[dst] = i; + registerLastChanged[src] = i; +#endif + } + continue; + } + opcode -= RANDOMX_FREQ_ISWAP_R; + + if (opcode < RANDOMX_FREQ_FSWAP_R + RANDOMX_FREQ_FADD_R + RANDOMX_FREQ_FADD_M + RANDOMX_FREQ_FSUB_R + RANDOMX_FREQ_FSUB_M + RANDOMX_FREQ_FSCAL_R + RANDOMX_FREQ_FMUL_R + RANDOMX_FREQ_FDIV_M + RANDOMX_FREQ_FSQRT_R) + { + // Mark FP instruction (src |= 0x20) + *(__global uint32_t*)(src_program + i) |= 0x20 << 8; + continue; + } + opcode -= RANDOMX_FREQ_FSWAP_R + RANDOMX_FREQ_FADD_R + RANDOMX_FREQ_FADD_M + RANDOMX_FREQ_FSUB_R + RANDOMX_FREQ_FSUB_M + RANDOMX_FREQ_FSCAL_R + RANDOMX_FREQ_FMUL_R + RANDOMX_FREQ_FDIV_M + RANDOMX_FREQ_FSQRT_R; + + if (opcode < RANDOMX_FREQ_CBRANCH) + { + const uint32_t creg = dst; +#if RANDOMX_PROGRAM_SIZE <= 256 + const uint32_t change = get_byte(registerLastChanged, dst); + const int32_t lastChanged = (get_byte(registerWasChanged, dst) == 0) ? -1 : (int32_t)(change); + + // Store condition register and branch target in CBRANCH instruction + *(__global uint32_t*)(src_program + i) = (src_inst.x & 0xFF0000FFU) | ((creg | ((lastChanged == -1) ? 0x90 : 0x10)) << 8) | (((uint32_t)(lastChanged) & 0xFF) << 16); +#else + const int32_t lastChanged = registerLastChanged[dst]; + + // Store condition register in CBRANCH instruction + *(__global uint32_t*)(src_program + i) = (src_inst.x & 0xFF0000FFU) | ((creg | 0x10) << 8); +#endif + + // Mark branch target instruction (src |= 0x40) + *(__global uint32_t*)(src_program + lastChanged + 1) |= 0x40 << 8; + +#if RANDOMX_PROGRAM_SIZE <= 256 + uint32_t tmp = i | (i << 8); + registerLastChanged = tmp | (tmp << 16); + registerLastChanged = registerLastChanged | (registerLastChanged << 32); + + registerWasChanged = 0x0101010101010101UL; +#else + registerLastChanged[0] = i; + registerLastChanged[1] = i; + registerLastChanged[2] = i; + registerLastChanged[3] = i; + registerLastChanged[4] = i; + registerLastChanged[5] = i; + registerLastChanged[6] = i; + registerLastChanged[7] = i; +#endif + } + } + + uint64_t registerLatency = 0; + uint64_t registerReadCycle = 0; + uint64_t registerLatencyFP = 0; + uint64_t registerReadCycleFP = 0; + uint32_t ScratchpadHighLatency = 0; + uint32_t ScratchpadLatency = 0; + + int32_t first_available_slot = 0; + int32_t first_allowed_slot_cfround = 0; + int32_t last_used_slot = -1; + int32_t last_memory_op_slot = -1; + + uint32_t num_slots_used = 0; + uint32_t num_instructions = 0; + + int32_t first_instruction_slot = -1; + bool first_instruction_fp = false; + + //if (global_index == 0) + //{ + // for (int j = 0; j < RANDOMX_PROGRAM_SIZE; ++j) + // { + // print_inst(src_program[j]); + // printf("\n"); + // } + // printf("\n"); + //} + +)===" +R"===( + + // Schedule instructions + bool update_branch_target_mark = false; + bool first_available_slot_is_branch_target = false; + for (uint32_t i = 0; i < RANDOMX_PROGRAM_SIZE; ++i) + { + const uint2 inst = src_program[i]; + + uint32_t opcode = inst.x & 0xff; + uint32_t dst = (inst.x >> 8) & 7; + const uint32_t src = (inst.x >> 16) & 7; + const uint32_t mod = (inst.x >> 24); + + bool is_branch_target = (inst.x & (0x40 << 8)) != 0; + if (is_branch_target) + { + // If an instruction is a branch target, we can't move it before any previous instructions + first_available_slot = last_used_slot + 1; + + // Mark this slot as a branch target + // Whatever instruction takes this slot will receive branch target flag + first_available_slot_is_branch_target = true; + } + + const uint32_t dst_latency = get_byte(registerLatency, dst); + const uint32_t src_latency = get_byte(registerLatency, src); + const uint32_t reg_read_latency = (dst_latency > src_latency) ? dst_latency : src_latency; + const uint32_t mem_read_latency = ((dst == src) && ((inst.y & ScratchpadL3Mask64) >= RANDOMX_SCRATCHPAD_L2)) ? ScratchpadHighLatency : ScratchpadLatency; + + uint32_t full_read_latency = mem_read_latency; + update_max(full_read_latency, reg_read_latency); + + uint32_t latency = 0; + bool is_memory_op = false; + bool is_memory_store = false; + bool is_nop = false; + bool is_branch = false; + bool is_swap = false; + bool is_src_read = true; + bool is_fp = false; + bool is_cfround = false; + + do { + if (opcode < RANDOMX_FREQ_IADD_RS) + { + latency = reg_read_latency; + break; + } + opcode -= RANDOMX_FREQ_IADD_RS; + + if (opcode < RANDOMX_FREQ_IADD_M) + { + latency = full_read_latency; + is_memory_op = true; + break; + } + opcode -= RANDOMX_FREQ_IADD_M; + + if (opcode < RANDOMX_FREQ_ISUB_R) + { + latency = reg_read_latency; + break; + } + opcode -= RANDOMX_FREQ_ISUB_R; + + if (opcode < RANDOMX_FREQ_ISUB_M) + { + latency = full_read_latency; + is_memory_op = true; + break; + } + opcode -= RANDOMX_FREQ_ISUB_M; + + if (opcode < RANDOMX_FREQ_IMUL_R) + { + latency = reg_read_latency; + break; + } + opcode -= RANDOMX_FREQ_IMUL_R; + + if (opcode < RANDOMX_FREQ_IMUL_M) + { + latency = full_read_latency; + is_memory_op = true; + break; + } + opcode -= RANDOMX_FREQ_IMUL_M; + + if (opcode < RANDOMX_FREQ_IMULH_R) + { + latency = reg_read_latency; + break; + } + opcode -= RANDOMX_FREQ_IMULH_R; + + if (opcode < RANDOMX_FREQ_IMULH_M) + { + latency = full_read_latency; + is_memory_op = true; + break; + } + opcode -= RANDOMX_FREQ_IMULH_M; + + if (opcode < RANDOMX_FREQ_ISMULH_R) + { + latency = reg_read_latency; + break; + } + opcode -= RANDOMX_FREQ_ISMULH_R; + + if (opcode < RANDOMX_FREQ_ISMULH_M) + { + latency = full_read_latency; + is_memory_op = true; + break; + } + opcode -= RANDOMX_FREQ_ISMULH_M; + + if (opcode < RANDOMX_FREQ_IMUL_RCP) + { + is_src_read = false; + if (inst.y & (inst.y - 1)) + latency = dst_latency; + else + is_nop = true; + break; + } + opcode -= RANDOMX_FREQ_IMUL_RCP; + + if (opcode < RANDOMX_FREQ_INEG_R) + { + is_src_read = false; + latency = dst_latency; + break; + } + opcode -= RANDOMX_FREQ_INEG_R; + + if (opcode < RANDOMX_FREQ_IXOR_R) + { + latency = reg_read_latency; + break; + } + opcode -= RANDOMX_FREQ_IXOR_R; + + if (opcode < RANDOMX_FREQ_IXOR_M) + { + latency = full_read_latency; + is_memory_op = true; + break; + } + opcode -= RANDOMX_FREQ_IXOR_M; + + if (opcode < RANDOMX_FREQ_IROR_R + RANDOMX_FREQ_IROL_R) + { + latency = reg_read_latency; + break; + } + opcode -= RANDOMX_FREQ_IROR_R + RANDOMX_FREQ_IROL_R; + + if (opcode < RANDOMX_FREQ_ISWAP_R) + { + is_swap = true; + if (dst != src) + latency = reg_read_latency; + else + is_nop = true; + break; + } + opcode -= RANDOMX_FREQ_ISWAP_R; + + if (opcode < RANDOMX_FREQ_FSWAP_R) + { + latency = get_byte(registerLatencyFP, dst); + is_fp = true; + is_src_read = false; + break; + } + opcode -= RANDOMX_FREQ_FSWAP_R; + + if (opcode < RANDOMX_FREQ_FADD_R) + { + dst %= RegisterCountFlt; + latency = get_byte(registerLatencyFP, dst); + is_fp = true; + is_src_read = false; + break; + } + opcode -= RANDOMX_FREQ_FADD_R; + + if (opcode < RANDOMX_FREQ_FADD_M) + { + dst %= RegisterCountFlt; + latency = get_byte(registerLatencyFP, dst); + update_max(latency, src_latency); + update_max(latency, ScratchpadLatency); + is_fp = true; + is_memory_op = true; + break; + } + opcode -= RANDOMX_FREQ_FADD_M; + + if (opcode < RANDOMX_FREQ_FSUB_R) + { + dst %= RegisterCountFlt; + latency = get_byte(registerLatencyFP, dst); + is_fp = true; + is_src_read = false; + break; + } + opcode -= RANDOMX_FREQ_FSUB_R; + + if (opcode < RANDOMX_FREQ_FSUB_M) + { + dst %= RegisterCountFlt; + latency = get_byte(registerLatencyFP, dst); + update_max(latency, src_latency); + update_max(latency, ScratchpadLatency); + is_fp = true; + is_memory_op = true; + break; + } + opcode -= RANDOMX_FREQ_FSUB_M; + + if (opcode < RANDOMX_FREQ_FSCAL_R) + { + dst %= RegisterCountFlt; + latency = get_byte(registerLatencyFP, dst); + is_fp = true; + is_src_read = false; + break; + } + opcode -= RANDOMX_FREQ_FSCAL_R; + + if (opcode < RANDOMX_FREQ_FMUL_R) + { + dst = (dst % RegisterCountFlt) + RegisterCountFlt; + latency = get_byte(registerLatencyFP, dst); + is_fp = true; + is_src_read = false; + break; + } + opcode -= RANDOMX_FREQ_FMUL_R; + + if (opcode < RANDOMX_FREQ_FDIV_M) + { + dst = (dst % RegisterCountFlt) + RegisterCountFlt; + latency = get_byte(registerLatencyFP, dst); + update_max(latency, src_latency); + update_max(latency, ScratchpadLatency); + is_fp = true; + is_memory_op = true; + break; + } + opcode -= RANDOMX_FREQ_FDIV_M; + + if (opcode < RANDOMX_FREQ_FSQRT_R) + { + dst = (dst % RegisterCountFlt) + RegisterCountFlt; + latency = get_byte(registerLatencyFP, dst); + is_fp = true; + is_src_read = false; + break; + } + opcode -= RANDOMX_FREQ_FSQRT_R; + + if (opcode < RANDOMX_FREQ_CBRANCH) + { + is_src_read = false; + is_branch = true; + latency = dst_latency; + + // We can't move CBRANCH before any previous instructions + first_available_slot = last_used_slot + 1; + break; + } + opcode -= RANDOMX_FREQ_CBRANCH; + + if (opcode < RANDOMX_FREQ_CFROUND) + { + latency = src_latency; + is_cfround = true; + break; + } + opcode -= RANDOMX_FREQ_CFROUND; + + if (opcode < RANDOMX_FREQ_ISTORE) + { + latency = reg_read_latency; + update_max(latency, (last_memory_op_slot + WORKERS_PER_HASH) / WORKERS_PER_HASH); + is_memory_op = true; + is_memory_store = true; + break; + } + opcode -= RANDOMX_FREQ_ISTORE; + + is_nop = true; + } while (false); + + if (is_nop) + { + if (is_branch_target) + { + // Mark next non-NOP instruction as the branch target instead of this NOP + update_branch_target_mark = true; + } + continue; + } + + if (update_branch_target_mark) + { + *(__global uint32_t*)(src_program + i) |= 0x40 << 8; + update_branch_target_mark = false; + is_branch_target = true; + } + + int32_t first_allowed_slot = first_available_slot; + update_max(first_allowed_slot, latency * WORKERS_PER_HASH); + if (is_cfround) + update_max(first_allowed_slot, first_allowed_slot_cfround); + else + update_max(first_allowed_slot, get_byte(is_fp ? registerReadCycleFP : registerReadCycle, dst) * WORKERS_PER_HASH); + + if (is_swap) + update_max(first_allowed_slot, get_byte(registerReadCycle, src) * WORKERS_PER_HASH); + + int32_t slot_to_use = last_used_slot + 1; + update_max(slot_to_use, first_allowed_slot); + + if (is_fp) + { + slot_to_use = -1; + for (int32_t j = first_allowed_slot; slot_to_use < 0; ++j) + { + if ((execution_plan[j] == 0) && (execution_plan[j + 1] == 0) && ((j + 1) % WORKERS_PER_HASH)) + { + bool blocked = false; + for (int32_t k = (j / WORKERS_PER_HASH) * WORKERS_PER_HASH; k < j; ++k) + { + if (execution_plan[k] || (k == first_instruction_slot)) + { + const uint32_t inst = src_program[execution_plan[k]].x; + + // If there is an integer instruction which is a branch target or a branch, or this FP instruction is a branch target itself, we can't reorder it to add more FP instructions to this cycle + if (((inst & (0x20 << 8)) == 0) && (((inst & (0x50 << 8)) != 0) || is_branch_target)) + { + blocked = true; + continue; + } + } + } + + if (!blocked) + { + for (int32_t k = (j / WORKERS_PER_HASH) * WORKERS_PER_HASH; k < j; ++k) + { + if (execution_plan[k] || (k == first_instruction_slot)) + { + const uint32_t inst = src_program[execution_plan[k]].x; + if ((inst & (0x20 << 8)) == 0) + { + execution_plan[j] = execution_plan[k]; + execution_plan[j + 1] = execution_plan[k + 1]; + if (first_instruction_slot == k) first_instruction_slot = j; + if (first_instruction_slot == k + 1) first_instruction_slot = j + 1; + slot_to_use = k; + break; + } + } + } + + if (slot_to_use < 0) + { + slot_to_use = j; + } + + break; + } + } + } + } + else + { + for (int32_t j = first_allowed_slot; j <= last_used_slot; ++j) + { + if (execution_plan[j] == 0) + { + slot_to_use = j; + break; + } + } + } + + if (i == 0) + { + first_instruction_slot = slot_to_use; + first_instruction_fp = is_fp; + } + + if (is_cfround) + { + first_allowed_slot_cfround = slot_to_use - (slot_to_use % WORKERS_PER_HASH) + WORKERS_PER_HASH; + } + + ++num_instructions; + + execution_plan[slot_to_use] = i; + ++num_slots_used; + + if (is_fp) + { + execution_plan[slot_to_use + 1] = i; + ++num_slots_used; + } + + const uint32_t next_latency = (slot_to_use / WORKERS_PER_HASH) + 1; + + if (is_src_read) + { + int32_t value = get_byte(registerReadCycle, src); + update_max(value, slot_to_use / WORKERS_PER_HASH); + set_byte(registerReadCycle, src, value); + } + + if (is_memory_op) + { + update_max(last_memory_op_slot, slot_to_use); + } + + if (is_cfround) + { + const uint32_t t = next_latency | (next_latency << 8); + registerLatencyFP = t | (t << 16); + registerLatencyFP = registerLatencyFP | (registerLatencyFP << 32); + } + else if (is_fp) + { + set_byte(registerLatencyFP, dst, next_latency); + + int32_t value = get_byte(registerReadCycleFP, dst); + update_max(value, slot_to_use / WORKERS_PER_HASH); + set_byte(registerReadCycleFP, dst, value); + } + else + { + if (!is_memory_store && !is_nop) + { + set_byte(registerLatency, dst, next_latency); + if (is_swap) + set_byte(registerLatency, src, next_latency); + + int32_t value = get_byte(registerReadCycle, dst); + update_max(value, slot_to_use / WORKERS_PER_HASH); + set_byte(registerReadCycle, dst, value); + } + + if (is_branch) + { + const uint32_t t = next_latency | (next_latency << 8); + registerLatency = t | (t << 16); + registerLatency = registerLatency | (registerLatency << 32); + } + + if (is_memory_store) + { + int32_t value = get_byte(registerReadCycle, dst); + update_max(value, slot_to_use / WORKERS_PER_HASH); + set_byte(registerReadCycle, dst, value); + ScratchpadLatency = (slot_to_use / WORKERS_PER_HASH) + 1; + if ((mod >> 4) >= StoreL3Condition) + ScratchpadHighLatency = (slot_to_use / WORKERS_PER_HASH) + 1; + } + } + + if (execution_plan[first_available_slot] || (first_available_slot == first_instruction_slot)) + { + if (first_available_slot_is_branch_target) + { + src_program[i].x |= 0x40 << 8; + first_available_slot_is_branch_target = false; + } + + if (is_fp) + ++first_available_slot; + + do { + ++first_available_slot; + } while ((first_available_slot < RANDOMX_PROGRAM_SIZE * WORKERS_PER_HASH) && (execution_plan[first_available_slot] != 0)); + } + + if (is_branch_target) + { + update_max(first_available_slot, is_fp ? (slot_to_use + 2) : (slot_to_use + 1)); + } + + update_max(last_used_slot, is_fp ? (slot_to_use + 1) : slot_to_use); + while (execution_plan[last_used_slot] || (last_used_slot == first_instruction_slot) || ((last_used_slot == first_instruction_slot + 1) && first_instruction_fp)) + { + ++last_used_slot; + } + --last_used_slot; + + if (is_fp && (last_used_slot >= first_allowed_slot_cfround)) + first_allowed_slot_cfround = last_used_slot + 1; + + //if (global_index == 0) + //{ + // printf("slot_to_use = %d, first_available_slot = %d, last_used_slot = %d\n", slot_to_use, first_available_slot, last_used_slot); + // for (int j = 0; j <= last_used_slot; ++j) + // { + // if (execution_plan[j] || (j == first_instruction_slot) || ((j == first_instruction_slot + 1) && first_instruction_fp)) + // { + // print_inst(src_program[execution_plan[j]]); + // printf(" | "); + // } + // else + // { + // printf(" | "); + // } + // if (((j + 1) % WORKERS_PER_HASH) == 0) printf("\n"); + // } + // printf("\n\n"); + //} + } + + //if (global_index == 0) + //{ + // printf("IPC = %.3f, WPC = %.3f, num_instructions = %u, num_slots_used = %u, first_instruction_slot = %d, last_used_slot = %d, registerLatency = %016llx, registerLatencyFP = %016llx \n", + // num_instructions / static_cast(last_used_slot / WORKERS_PER_HASH + 1), + // num_slots_used / static_cast(last_used_slot / WORKERS_PER_HASH + 1), + // num_instructions, + // num_slots_used, + // first_instruction_slot, + // last_used_slot, + // registerLatency, + // registerLatencyFP + // ); + + // //for (int j = 0; j < RANDOMX_PROGRAM_SIZE; ++j) + // //{ + // // print_inst(src_program[j]); + // // printf("\n"); + // //} + // //printf("\n"); + + // for (int j = 0; j <= last_used_slot; ++j) + // { + // if (execution_plan[j] || (j == first_instruction_slot) || ((j == first_instruction_slot + 1) && first_instruction_fp)) + // { + // print_inst(src_program[execution_plan[j]]); + // printf(" | "); + // } + // else + // { + // printf(" | "); + // } + // if (((j + 1) % WORKERS_PER_HASH) == 0) printf("\n"); + // } + // printf("\n\n"); + //} + + //atomicAdd((uint32_t*)num_vm_cycles, (last_used_slot / WORKERS_PER_HASH) + 1); + //atomicAdd((uint32_t*)(num_vm_cycles) + 1, num_slots_used); + + uint32_t ma = (uint32_t)(entropy[8]) & CacheLineAlignMask; + uint32_t mx = (uint32_t)(entropy[10]) & CacheLineAlignMask; + + uint32_t addressRegisters = (uint32_t)(entropy[12]); + addressRegisters = ((addressRegisters & 1) | (((addressRegisters & 2) ? 3U : 2U) << 8) | (((addressRegisters & 4) ? 5U : 4U) << 16) | (((addressRegisters & 8) ? 7U : 6U) << 24)) * sizeof(uint64_t); + + uint32_t datasetOffset = (entropy[13] & DatasetExtraItems) * CacheLineSize; + + ulong2 eMask = *(__global ulong2*)(entropy + 14); + eMask.x = getFloatMask(eMask.x); + eMask.y = getFloatMask(eMask.y); + + ((__global uint32_t*)(R + 16))[0] = ma; + ((__global uint32_t*)(R + 16))[1] = mx; + ((__global uint32_t*)(R + 16))[2] = addressRegisters; + ((__global uint32_t*)(R + 16))[3] = datasetOffset; + ((__global ulong2*)(R + 18))[0] = eMask; + + __global uint32_t* imm_buf = (__global uint32_t*)(R + REGISTERS_SIZE / sizeof(uint64_t)); + uint32_t imm_index = 0; + int32_t imm_index_fscal_r = -1; + __global uint32_t* compiled_program = (__global uint32_t*)(R + (REGISTERS_SIZE + IMM_BUF_SIZE) / sizeof(uint64_t)); + +)===" +R"===( + + // Generate opcodes for execute_vm + int32_t branch_target_slot = -1; + int32_t k = -1; + for (int32_t i = 0; i <= last_used_slot; ++i) + { + if (!(execution_plan[i] || (i == first_instruction_slot) || ((i == first_instruction_slot + 1) && first_instruction_fp))) + continue; + + uint32_t num_workers = 1; + uint32_t num_fp_insts = 0; + while ((i + num_workers <= last_used_slot) && ((i + num_workers) % WORKERS_PER_HASH) && (execution_plan[i + num_workers] || (i + num_workers == first_instruction_slot) || ((i + num_workers == first_instruction_slot + 1) && first_instruction_fp))) + { + if ((num_workers & 1) && ((src_program[execution_plan[i + num_workers]].x & (0x20 << 8)) != 0)) + ++num_fp_insts; + ++num_workers; + } + + //if (global_index == 0) + // printf("i = %d, num_workers = %u, num_fp_insts = %u\n", i, num_workers, num_fp_insts); + + num_workers = ((num_workers - 1) << NUM_INSTS_OFFSET) | (num_fp_insts << NUM_FP_INSTS_OFFSET); + + const uint2 src_inst = src_program[execution_plan[i]]; + uint2 inst = src_inst; + + uint32_t opcode = inst.x & 0xff; + const uint32_t dst = (inst.x >> 8) & 7; + const uint32_t src = (inst.x >> 16) & 7; + const uint32_t mod = (inst.x >> 24); + + const bool is_fp = (src_inst.x & (0x20 << 8)) != 0; + if (is_fp && ((i & 1) == 0)) + ++i; + + const bool is_branch_target = (src_inst.x & (0x40 << 8)) != 0; + if (is_branch_target && (branch_target_slot < 0)) + branch_target_slot = k; + + ++k; + + inst.x = INST_NOP; + + if (opcode < RANDOMX_FREQ_IADD_RS) + { + const uint32_t shift = (mod >> 2) % 4; + + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (shift << SHIFT_OFFSET); + + if (dst != RegisterNeedsDisplacement) + { + // Encode regular ADD (opcode 1) + inst.x |= (1 << OPCODE_OFFSET); + } + else + { + // Encode ADD with src and imm32 (opcode 0) + inst.x |= imm_index << IMM_OFFSET; + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = inst.y; + } + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_IADD_RS; + + if (opcode < RANDOMX_FREQ_IADD_M) + { + const uint32_t location = (src == dst) ? 3 : ((mod % 4) ? 1 : 2); + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (1 << LOC_OFFSET) | (1 << OPCODE_OFFSET); + inst.x |= imm_index << IMM_OFFSET; + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = (inst.y & 0xFC1FFFFFU) | (((location == 1) ? LOC_L1 : ((location == 2) ? LOC_L2 : LOC_L3)) << 21); + else + inst.x = INST_NOP; + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_IADD_M; + + if (opcode < RANDOMX_FREQ_ISUB_R) + { + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (1 << OPCODE_OFFSET) | (1 << NEGATIVE_SRC_OFFSET); + if (src == dst) + { + inst.x |= (imm_index << IMM_OFFSET) | (1 << SRC_IS_IMM32_OFFSET); + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = inst.y; + } + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_ISUB_R; + + if (opcode < RANDOMX_FREQ_ISUB_M) + { + const uint32_t location = (src == dst) ? 3 : ((mod % 4) ? 1 : 2); + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (1 << LOC_OFFSET) | (1 << OPCODE_OFFSET) | (1 << NEGATIVE_SRC_OFFSET); + inst.x |= imm_index << IMM_OFFSET; + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = (inst.y & 0xFC1FFFFFU) | (((location == 1) ? LOC_L1 : ((location == 2) ? LOC_L2 : LOC_L3)) << 21); + else + inst.x = INST_NOP; + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_ISUB_M; + + if (opcode < RANDOMX_FREQ_IMUL_R) + { + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (2 << OPCODE_OFFSET); + if (src == dst) + { + inst.x |= (imm_index << IMM_OFFSET) | (1 << SRC_IS_IMM32_OFFSET); + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = inst.y; + } + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_IMUL_R; + + if (opcode < RANDOMX_FREQ_IMUL_M) + { + const uint32_t location = (src == dst) ? 3 : ((mod % 4) ? 1 : 2); + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (1 << LOC_OFFSET) | (2 << OPCODE_OFFSET); + inst.x |= imm_index << IMM_OFFSET; + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = (inst.y & 0xFC1FFFFFU) | (((location == 1) ? LOC_L1 : ((location == 2) ? LOC_L2 : LOC_L3)) << 21); + else + inst.x = INST_NOP; + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_IMUL_M; + + if (opcode < RANDOMX_FREQ_IMULH_R) + { + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (6 << OPCODE_OFFSET); + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_IMULH_R; + + if (opcode < RANDOMX_FREQ_IMULH_M) + { + const uint32_t location = (src == dst) ? 3 : ((mod % 4) ? 1 : 2); + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (1 << LOC_OFFSET) | (6 << OPCODE_OFFSET); + inst.x |= imm_index << IMM_OFFSET; + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = (inst.y & 0xFC1FFFFFU) | (((location == 1) ? LOC_L1 : ((location == 2) ? LOC_L2 : LOC_L3)) << 21); + else + inst.x = INST_NOP; + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_IMULH_M; + + if (opcode < RANDOMX_FREQ_ISMULH_R) + { + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (4 << OPCODE_OFFSET); + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_ISMULH_R; + + if (opcode < RANDOMX_FREQ_ISMULH_M) + { + const uint32_t location = (src == dst) ? 3 : ((mod % 4) ? 1 : 2); + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (1 << LOC_OFFSET) | (4 << OPCODE_OFFSET); + inst.x |= imm_index << IMM_OFFSET; + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = (inst.y & 0xFC1FFFFFU) | (((location == 1) ? LOC_L1 : ((location == 2) ? LOC_L2 : LOC_L3)) << 21); + else + inst.x = INST_NOP; + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_ISMULH_M; + + if (opcode < RANDOMX_FREQ_IMUL_RCP) + { + const uint64_t r = imul_rcp_value(inst.y); + if (r == 1) + { + *(compiled_program++) = INST_NOP | num_workers; + continue; + } + + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (2 << OPCODE_OFFSET); + inst.x |= (imm_index << IMM_OFFSET) | (1 << SRC_IS_IMM64_OFFSET); + + if (imm_index < IMM_INDEX_COUNT - 1) + { + imm_buf[imm_index] = ((const uint32_t*)&r)[0]; + imm_buf[imm_index + 1] = ((const uint32_t*)&r)[1]; + imm_index += 2; + } + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_IMUL_RCP; + + if (opcode < RANDOMX_FREQ_INEG_R) + { + inst.x = (dst << DST_OFFSET) | (5 << OPCODE_OFFSET); + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_INEG_R; + + if (opcode < RANDOMX_FREQ_IXOR_R) + { + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (3 << OPCODE_OFFSET); + if (src == dst) + { + inst.x |= (imm_index << IMM_OFFSET) | (1 << SRC_IS_IMM32_OFFSET); + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = inst.y; + } + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_IXOR_R; + + if (opcode < RANDOMX_FREQ_IXOR_M) + { + const uint32_t location = (src == dst) ? 3 : ((mod % 4) ? 1 : 2); + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (1 << LOC_OFFSET) | (3 << OPCODE_OFFSET); + inst.x |= imm_index << IMM_OFFSET; + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = (inst.y & 0xFC1FFFFFU) | (((location == 1) ? LOC_L1 : ((location == 2) ? LOC_L2 : LOC_L3)) << 21); + else + inst.x = INST_NOP; + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_IXOR_M; + + if (opcode < RANDOMX_FREQ_IROR_R + RANDOMX_FREQ_IROL_R) + { + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (7 << OPCODE_OFFSET); + if (src == dst) + { + inst.x |= (imm_index << IMM_OFFSET) | (1 << SRC_IS_IMM32_OFFSET); + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = inst.y; + } + if (opcode >= RANDOMX_FREQ_IROR_R) + { + inst.x |= (1 << NEGATIVE_SRC_OFFSET); + } + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_IROR_R + RANDOMX_FREQ_IROL_R; + + if (opcode < RANDOMX_FREQ_ISWAP_R) + { + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (8 << OPCODE_OFFSET); + + *(compiled_program++) = ((src != dst) ? inst.x : INST_NOP) | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_ISWAP_R; + + if (opcode < RANDOMX_FREQ_FSWAP_R) + { + inst.x = (dst << DST_OFFSET) | (11 << OPCODE_OFFSET); + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_FSWAP_R; + + if (opcode < RANDOMX_FREQ_FADD_R) + { + inst.x = ((dst % RegisterCountFlt) << DST_OFFSET) | ((src % RegisterCountFlt) << (SRC_OFFSET + 1)) | (12 << OPCODE_OFFSET); + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_FADD_R; + + if (opcode < RANDOMX_FREQ_FADD_M) + { + const uint32_t location = (mod % 4) ? 1 : 2; + inst.x = ((dst % RegisterCountFlt) << DST_OFFSET) | (src << SRC_OFFSET) | (1 << LOC_OFFSET) | (12 << OPCODE_OFFSET); + inst.x |= imm_index << IMM_OFFSET; + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = (inst.y & 0xFC1FFFFFU) | (((location == 1) ? LOC_L1 : ((location == 2) ? LOC_L2 : LOC_L3)) << 21); + else + inst.x = INST_NOP; + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_FADD_M; + + if (opcode < RANDOMX_FREQ_FSUB_R) + { + inst.x = ((dst % RegisterCountFlt) << DST_OFFSET) | ((src % RegisterCountFlt) << (SRC_OFFSET + 1)) | (12 << OPCODE_OFFSET) | (1 << NEGATIVE_SRC_OFFSET); + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_FSUB_R; + + if (opcode < RANDOMX_FREQ_FSUB_M) + { + const uint32_t location = (mod % 4) ? 1 : 2; + inst.x = ((dst % RegisterCountFlt) << DST_OFFSET) | (src << SRC_OFFSET) | (1 << LOC_OFFSET) | (12 << OPCODE_OFFSET) | (1 << NEGATIVE_SRC_OFFSET); + inst.x |= imm_index << IMM_OFFSET; + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = (inst.y & 0xFC1FFFFFU) | (((location == 1) ? LOC_L1 : ((location == 2) ? LOC_L2 : LOC_L3)) << 21); + else + inst.x = INST_NOP; + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_FSUB_M; + + if (opcode < RANDOMX_FREQ_FSCAL_R) + { + inst.x = ((dst % RegisterCountFlt) << DST_OFFSET) | (1 << SRC_IS_IMM64_OFFSET) | (3 << OPCODE_OFFSET); + if (imm_index_fscal_r >= 0) + { + inst.x |= (imm_index_fscal_r << IMM_OFFSET); + } + else + { + imm_index_fscal_r = imm_index; + inst.x |= (imm_index << IMM_OFFSET); + + if (imm_index < IMM_INDEX_COUNT - 1) + { + imm_buf[imm_index] = 0; + imm_buf[imm_index + 1] = 0x80F00000UL; + imm_index += 2; + } + } + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_FSCAL_R; + + if (opcode < RANDOMX_FREQ_FMUL_R) + { + inst.x = (((dst % RegisterCountFlt) + RegisterCountFlt) << DST_OFFSET) | ((src % RegisterCountFlt) << (SRC_OFFSET + 1)) | (1 << SHIFT_OFFSET) | (12 << OPCODE_OFFSET); + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_FMUL_R; + + if (opcode < RANDOMX_FREQ_FDIV_M) + { + const uint32_t location = (mod % 4) ? 1 : 2; + inst.x = (((dst % RegisterCountFlt) + RegisterCountFlt) << DST_OFFSET) | (src << SRC_OFFSET) | (1 << LOC_OFFSET) | (15 << OPCODE_OFFSET); + inst.x |= imm_index << IMM_OFFSET; + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = (inst.y & 0xFC1FFFFFU) | (((location == 1) ? LOC_L1 : ((location == 2) ? LOC_L2 : LOC_L3)) << 21); + else + inst.x = INST_NOP; + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_FDIV_M; + + if (opcode < RANDOMX_FREQ_FSQRT_R) + { + inst.x = (((dst % RegisterCountFlt) + RegisterCountFlt) << DST_OFFSET) | (14 << OPCODE_OFFSET); + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_FSQRT_R; + + if (opcode < RANDOMX_FREQ_CBRANCH) + { + inst.x = (dst << DST_OFFSET) | (9 << OPCODE_OFFSET); + inst.x |= (imm_index << IMM_OFFSET); + + const uint32_t cshift = (mod >> 4) + ConditionOffset; + + uint32_t imm = inst.y | (1U << cshift); + if (cshift > 0) + imm &= ~(1U << (cshift - 1)); + + if (imm_index < IMM_INDEX_COUNT - 1) + { + imm_buf[imm_index] = imm; + imm_buf[imm_index + 1] = cshift | ((uint32_t)(branch_target_slot) << 5); + imm_index += 2; + } + else + { + // Data doesn't fit, skip it + inst.x = INST_NOP; + } + + branch_target_slot = -1; + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_CBRANCH; + + if (opcode < RANDOMX_FREQ_CFROUND) + { + inst.x = (src << SRC_OFFSET) | (13 << OPCODE_OFFSET) | ((inst.y & 63) << IMM_OFFSET); + + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_CFROUND; + + if (opcode < RANDOMX_FREQ_ISTORE) + { + const uint32_t location = ((mod >> 4) >= StoreL3Condition) ? 3 : ((mod % 4) ? 1 : 2); + inst.x = (dst << DST_OFFSET) | (src << SRC_OFFSET) | (1 << LOC_OFFSET) | (10 << OPCODE_OFFSET); + inst.x |= imm_index << IMM_OFFSET; + if (imm_index < IMM_INDEX_COUNT) + imm_buf[imm_index++] = (inst.y & 0xFC1FFFFFU) | (((location == 1) ? LOC_L1 : ((location == 2) ? LOC_L2 : LOC_L3)) << 21); + else + inst.x = INST_NOP; + *(compiled_program++) = inst.x | num_workers; + continue; + } + opcode -= RANDOMX_FREQ_ISTORE; + + *(compiled_program++) = inst.x | num_workers; + } + + ((__global uint32_t*)(R + 20))[0] = (uint32_t)(compiled_program - (__global uint32_t*)(R + (REGISTERS_SIZE + IMM_BUF_SIZE) / sizeof(uint64_t))); + } +} + +void load_buffer(__local uint64_t *dst_buf, size_t N, __global const void* src_buf) +{ + uint32_t i = get_local_id(0) * sizeof(uint64_t); + const uint32_t step = get_local_size(0) * sizeof(uint64_t); + __global const uint8_t* src = ((__global const uint8_t*)src_buf) + get_group_id(0) * sizeof(uint64_t) * N + i; + __local uint8_t* dst = ((__local uint8_t*)dst_buf) + i; + while (i < sizeof(uint64_t) * N) + { + *(__local uint64_t*)(dst) = *(__global uint64_t*)(src); + src += step; + dst += step; + i += step; + } +} + +double load_F_E_groups(int value, uint64_t andMask, uint64_t orMask) +{ + double t = convert_double_rtn(value); + uint64_t x = as_ulong(t); + x &= andMask; + x |= orMask; + return as_double(x); +} + +)===" +R"===( + +// You're one ugly motherfucker! +double fma_soft(double a, double b, double c, uint32_t rounding_mode) +{ + if (rounding_mode == 0) + return fma(a, b, c); + + if ((a == 0.0) || (b == 0.0)) + return c; + + if (b == 1.0) + { + if (c == 0.0) + return a; + + if (c == -a) + { + const uint64_t minus_zero = 1UL << 63; + return (rounding_mode == 1) ? as_double(minus_zero) : 0.0; + } + } + + const uint64_t mantissa_size = 52; + const uint64_t mantissa_mask = (1UL << mantissa_size) - 1; + const uint64_t mantissa_high_bit = 1UL << mantissa_size; + + const uint64_t exponent_size = 11; + const uint64_t exponent_mask = (1 << exponent_size) - 1; + + const uint32_t exponent_a = (as_uint2(a).y >> 20) & exponent_mask; + const uint32_t exponent_b = (as_uint2(b).y >> 20) & exponent_mask; + const uint32_t exponent_c = (as_uint2(c).y >> 20) & exponent_mask; + + if ((exponent_a == 2047) || (exponent_b == 2047) || (exponent_c == 2047)) + { + const uint64_t inf = 2047UL << 52; + return as_double(inf); + } + + const uint64_t mantissa_a = (as_ulong(a) & mantissa_mask) | mantissa_high_bit; + const uint64_t mantissa_b = (as_ulong(b) & mantissa_mask) | mantissa_high_bit; + const uint64_t mantissa_c = (as_ulong(c) & mantissa_mask) | mantissa_high_bit; + + const uint32_t sign_a = as_uint2(a).y >> 31; + const uint32_t sign_b = as_uint2(b).y >> 31; + const uint32_t sign_c = as_uint2(c).y >> 31; + + uint64_t mul_result[2]; + mul_result[0] = mantissa_a * mantissa_b; + mul_result[1] = mul_hi(mantissa_a, mantissa_b); + + uint32_t exp_correction = mul_result[1] >> 41; + uint32_t exponent_mul_result = exponent_a + exponent_b + exp_correction - 1023; + uint32_t sign_mul_result = sign_a ^ sign_b; + + if (exponent_mul_result >= 2047) + { + const uint64_t inf_rnd = (2047UL << 52) - (rounding_mode & 1); + return as_double(inf_rnd); + } + + uint64_t fma_result[2]; + uint64_t t[2]; + uint32_t exponent_fma_result; + + if (exponent_mul_result >= exponent_c) + { + uint32_t shift = 23 - exp_correction; + fma_result[0] = mul_result[0] << shift; + fma_result[1] = (mul_result[1] << shift) | (mul_result[0] >> (64 - shift)); + + int32_t shift2 = (127 - 52) + (int32_t)(exponent_c - exponent_mul_result); + + if (shift2 >= 0) + { + if (shift2 >= 64) + { + t[0] = 0; + t[1] = mantissa_c << (shift2 - 64); + } + else + { + t[0] = mantissa_c << shift2; + t[1] = shift2 ? (mantissa_c >> (64 - shift2)) : 0; + } + } + else + { + t[0] = (shift2 < -52) ? 0 : (mantissa_c >> (-shift2)); + t[1] = 0; + if ((t[0] == 0) && (c != 0.0)) + t[0] = 1; + } + + exponent_fma_result = exponent_mul_result; + } + else + { + t[0] = 0; + t[1] = mantissa_c << 11; + + int32_t shift2 = (127 - 104 - exp_correction) + (int32_t)(exponent_mul_result - exponent_c); + if (shift2 >= 0) + { + fma_result[0] = mul_result[0] << shift2; + fma_result[1] = (mul_result[1] << shift2) | (shift2 ? (mul_result[0] >> (64 - shift2)) : 0); + } + else + { + shift2 = -shift2; + if (shift2 >= 64) + { + shift2 -= 64; + fma_result[0] = (shift2 < 64) ? (mul_result[1] >> shift2) : 0; + fma_result[1] = 0; + if (fma_result[0] == 0) + fma_result[0] = 1; + } + else + { + fma_result[0] = (mul_result[0] >> shift2) | (mul_result[1] << (64 - shift2)); + fma_result[1] = mul_result[1] >> shift2; + } + } + + exponent_fma_result = exponent_c; + } + + uint32_t sign_fma_result; + + if (sign_mul_result == sign_c) + { + fma_result[0] += t[0]; + fma_result[1] += t[1] + ((fma_result[0] < t[0]) ? 1 : 0); + + exp_correction = (fma_result[1] < t[1]) ? 1 : 0; + sign_fma_result = sign_mul_result; + } + else + { + const uint32_t borrow = (fma_result[0] < t[0]) ? 1 : 0; + fma_result[0] -= t[0]; + + t[1] += borrow; + const uint32_t change_sign = (fma_result[1] < t[1]) ? 1 : 0; + fma_result[1] -= t[1]; + + sign_fma_result = sign_mul_result ^ change_sign; + if (change_sign) + { + fma_result[0] = -(int64_t)(fma_result[0]); + fma_result[1] = ~fma_result[1]; + fma_result[1] += fma_result[0] ? 0 : 1; + } + + if (fma_result[1] == 0) + { + if (fma_result[0] == 0) + return 0.0; + + exponent_fma_result -= 64; + fma_result[1] = fma_result[0]; + fma_result[0] = 0; + } + + const uint32_t index = clz(fma_result[1]); + if (index) + { + exponent_fma_result -= index; + fma_result[1] = (fma_result[1] << index) | (fma_result[0] >> (64 - index)); + } + + exp_correction = 0; + } + + const uint32_t shift = 11 + exp_correction; + const uint32_t round_up = (fma_result[0] || (fma_result[1] & ((1 << shift) - 1))) ? 1 : 0; + + fma_result[1] >>= shift; + fma_result[1] &= mantissa_mask; + if (rounding_mode + sign_fma_result == 2) + { + fma_result[1] += round_up; + if (fma_result[1] == mantissa_high_bit) + { + fma_result[1] = 0; + ++exponent_fma_result; + } + } + fma_result[1] |= (uint64_t)(exponent_fma_result + exp_correction) << mantissa_size; + fma_result[1] |= (uint64_t)(sign_fma_result) << 63; + + return as_double(fma_result[1]); +} + +double div_rnd(double a, double b, uint32_t fprc) +{ + double y0 = 1.0 / b; + + // Do 1 Newton-Raphson iteration to get correct rounding + const double t0 = a * y0; + const double t1 = fma(-b, t0, a); + double result = fma_soft(y0, t1, t0, fprc); + + // Check for infinity/NaN + const uint64_t inf = 2047UL << 52; + const uint64_t inf_rnd = inf - (fprc & 1); + + if (((as_ulong(result) >> 52) & 2047) == 2047) result = as_double(inf_rnd); + if (as_ulong(a) == inf) result = a; + + return (a == b) ? 1.0 : result; +} + +double sqrt_rnd(double x, uint32_t fprc) +{ + double y0 = rsqrt(x); + + // First Newton-Raphson iteration + double t0 = y0 * x; + double t1 = y0 * -0.5; + t1 = fma(t1, t0, 0.5); // 0.5 * (1.0 - y0 * y0 * x) + const double y1_x = fma(t0, t1, t0); // y1 * x = 0.5 * y0 * x * (3.0 - y0 * y0 * x) + + // Second Newton-Raphson iteration + y0 *= 0.5; + y0 = fma(y0, t1, y0); // 0.5 * y1 + t1 = fma(-y1_x, y1_x, x); // x * (1.0 - x * y1 * y1) + + double result = fma_soft(t1, y0, y1_x, fprc); // x * 0.5 * y1 * (3.0 - x * y1 * y1) + + // Check for infinity + if (*((uint64_t*) &x) == (2047UL << 52)) result = x; + + return result; +} + +uint32_t inner_loop( + const uint32_t program_length, + __local const uint32_t* compiled_program, + const int32_t sub, + __global uint8_t* scratchpad, + const uint32_t fp_reg_offset, + const uint32_t fp_reg_group_A_offset, + __local uint64_t* R, + __local uint32_t* imm_buf, + const uint32_t batch_size, + uint32_t fprc, + const uint32_t fp_workers_mask, + const uint64_t xexponentMask, + const uint32_t workers_mask +) +{ + const int32_t sub2 = sub >> 1; + imm_buf[IMM_INDEX_COUNT + 1] = fprc; + + #pragma unroll(1) + for (int32_t ip = 0; ip < program_length;) + { + imm_buf[IMM_INDEX_COUNT] = ip; + + uint32_t inst = compiled_program[ip]; + const int32_t num_workers = (inst >> NUM_INSTS_OFFSET) & (WORKERS_PER_HASH - 1); + const int32_t num_fp_insts = (inst >> NUM_FP_INSTS_OFFSET) & (WORKERS_PER_HASH - 1); + const int32_t num_insts = num_workers - num_fp_insts; + + if (sub <= num_workers) + { + const int32_t inst_offset = sub - num_fp_insts; + const bool is_fp = inst_offset < num_fp_insts; + inst = compiled_program[ip + (is_fp ? sub2 : inst_offset)]; + //if ((idx == 0) && (ic == 0)) + //{ + // printf("num_fp_insts = %u, sub = %u, ip = %u, inst = %08x\n", num_fp_insts, sub, ip + ((sub < num_fp_insts * 2) ? (sub / 2) : (sub - num_fp_insts)), inst); + //} + + //asm("// INSTRUCTION DECODING BEGIN"); + + uint32_t opcode = (inst >> OPCODE_OFFSET) & 15; + const uint32_t location = (inst >> LOC_OFFSET) & 1; + + const uint32_t reg_size_shift = is_fp ? 4 : 3; + const uint32_t reg_base_offset = is_fp ? fp_reg_offset : 0; + const uint32_t reg_base_src_offset = is_fp ? fp_reg_group_A_offset : 0; + + uint32_t dst_offset = (inst >> DST_OFFSET) & 7; + dst_offset = reg_base_offset + (dst_offset << reg_size_shift); + + uint32_t src_offset = (inst >> SRC_OFFSET) & 7; + src_offset = (src_offset << 3) + (location ? 0 : reg_base_src_offset); + + __local uint64_t* dst_ptr = (__local uint64_t*)((__local uint8_t*)(R) + dst_offset); + __local uint64_t* src_ptr = (__local uint64_t*)((__local uint8_t*)(R) + src_offset); + + const uint32_t imm_offset = (inst >> IMM_OFFSET) & 255; + __local const uint32_t* imm_ptr = imm_buf + imm_offset; + + uint64_t dst = *dst_ptr; + uint64_t src = *src_ptr; + uint2 imm; + imm.x = imm_ptr[0]; + imm.y = imm_ptr[1]; + + //asm("// INSTRUCTION DECODING END"); + + if (location) + { + //asm("// SCRATCHPAD ACCESS BEGIN"); + + const uint32_t loc_shift = (imm.x >> 21) & 31; + const uint32_t mask = (0xFFFFFFFFU >> loc_shift) - 7; + + const bool is_read = (opcode != 10); + uint32_t addr = is_read ? ((loc_shift == LOC_L3) ? 0 : (uint32_t)(src)) : (uint32_t)(dst); + addr += (int32_t)(imm.x); + addr &= mask; + + __global uint64_t* ptr = (__global uint64_t*)(scratchpad + addr); + + if (is_read) + { + src = *ptr; + } + else + { + *ptr = src; + goto execution_end; + } + + //asm("// SCRATCHPAD ACCESS END"); + } + + { + //asm("// EXECUTION BEGIN"); + + if (inst & (1 << SRC_IS_IMM32_OFFSET)) src = (uint64_t)((int64_t)((int32_t)(imm.x))); + + // Check instruction opcodes (most frequent instructions come first) + if (opcode <= 3) + { + //asm("// IADD_RS, IADD_M, ISUB_R, ISUB_M, IMUL_R, IMUL_M, IMUL_RCP, IXOR_R, IXOR_M, FSCAL_R (109/256) ------>"); + if (inst & (1 << NEGATIVE_SRC_OFFSET)) src = (uint64_t)(-(int64_t)(src)); + if (opcode == 0) dst += (int32_t)(imm.x); + const uint32_t shift = (inst >> SHIFT_OFFSET) & 3; + if (opcode < 2) dst += src << shift; + const uint64_t imm64 = *((uint64_t*) &imm); + if (inst & (1 << SRC_IS_IMM64_OFFSET)) src = imm64; + if (opcode == 2) dst *= src; + if (opcode == 3) dst ^= src; + //asm("// <------ IADD_RS, IADD_M, ISUB_R, ISUB_M, IMUL_R, IMUL_M, IMUL_RCP, IXOR_R, IXOR_M, FSCAL_R (109/256)"); + } + else if (opcode == 12) + { + //asm("// FADD_R, FADD_M, FSUB_R, FSUB_M, FMUL_R (74/256) ------>"); + + if (location) src = as_ulong(convert_double_rtn((int32_t)(src >> ((sub & 1) * 32)))); + if (inst & (1 << NEGATIVE_SRC_OFFSET)) src ^= 0x8000000000000000UL; + + const bool is_mul = (inst & (1 << SHIFT_OFFSET)) != 0; + const double a = as_double(dst); + const double b = as_double(src); + + dst = as_ulong(fma_soft(a, is_mul ? b : 1.0, is_mul ? 0.0 : b, fprc)); + + //asm("// <------ FADD_R, FADD_M, FSUB_R, FSUB_M, FMUL_R (74/256)"); + } + else if (opcode == 9) + { + //asm("// CBRANCH (16/256) ------>"); + dst += (int32_t)(imm.x); + if (((uint32_t)(dst) & (ConditionMask << (imm.y & 31))) == 0) + { + imm_buf[IMM_INDEX_COUNT] = (uint32_t)(((int32_t)(imm.y) >> 5) - num_insts); + } + //asm("// <------ CBRANCH (16/256)"); + } + else if (opcode == 7) + { + //asm("// IROR_R, IROL_R (10/256) ------>"); + uint32_t shift1 = src & 63; +#if RANDOMX_FREQ_IROL_R > 0 + const uint32_t shift2 = 64 - shift1; + const bool is_rol = (inst & (1 << NEGATIVE_SRC_OFFSET)); + dst = (dst >> (is_rol ? shift2 : shift1)) | (dst << (is_rol ? shift1 : shift2)); +#else + dst = (dst >> shift1) | (dst << (64 - shift1)); +#endif + //asm("// <------ IROR_R, IROL_R (10/256)"); + } + else if (opcode == 14) + { + //asm("// FSQRT_R (6/256) ------>"); + dst = as_ulong(sqrt_rnd(as_double(dst), fprc)); + //asm("// <------ FSQRT_R (6/256)"); + } + else if (opcode == 6) + { + //asm("// IMULH_R, IMULH_M (5/256) ------>"); + dst = mul_hi(dst, src); + //asm("// <------ IMULH_R, IMULH_M (5/256)"); + } + else if (opcode == 4) + { + //asm("// ISMULH_R, ISMULH_M (5/256) ------>"); + dst = (uint64_t)(mul_hi((int64_t)(dst), (int64_t)(src))); + //asm("// <------ ISMULH_R, ISMULH_M (5/256)"); + } + else if (opcode == 11) + { + //asm("// FSWAP_R (4/256) ------>"); + dst = *(__local uint64_t*)((__local uint8_t*)(R) + (dst_offset ^ 8)); + //asm("// <------ FSWAP_R (4/256)"); + } + else if (opcode == 8) + { + //asm("// ISWAP_R (4/256) ------>"); + *src_ptr = dst; + dst = src; + //asm("// <------ ISWAP_R (4/256)"); + } + else if (opcode == 15) + { + //asm("// FDIV_M (4/256) ------>"); + src = as_ulong(convert_double_rtn((int32_t)(src >> ((sub & 1) * 32)))); + src &= dynamicMantissaMask; + src |= xexponentMask; + dst = as_ulong(div_rnd(as_double(dst), as_double(src), fprc)); + //asm("// <------ FDIV_M (4/256)"); + } + else if (opcode == 5) + { + //asm("// INEG_R (2/256) ------>"); + dst = (uint64_t)(-(int64_t)(dst)); + //asm("// <------ INEG_R (2/256)"); + } + // CFROUND check will be skipped and removed entirely by the compiler if ROUNDING_MODE >= 0 + else if (ROUNDING_MODE < 0) + { + //asm("// CFROUND (1/256) ------>"); + imm_buf[IMM_INDEX_COUNT + 1] = ((src >> imm_offset) | (src << (64 - imm_offset))) & 3; + //asm("// <------ CFROUND (1/256)"); + goto execution_end; + } + + *dst_ptr = dst; + //asm("// EXECUTION END"); + } + } + + execution_end: + { + //asm("// SYNCHRONIZATION OF INSTRUCTION POINTER AND ROUNDING MODE BEGIN"); + + barrier(CLK_LOCAL_MEM_FENCE); + ip = imm_buf[IMM_INDEX_COUNT]; + fprc = imm_buf[IMM_INDEX_COUNT + 1]; + + //asm("// SYNCHRONIZATION OF INSTRUCTION POINTER AND ROUNDING MODE END"); + + ip += num_insts + 1; + } + } + + return fprc; +} + +)===" +R"===( + +#if WORKERS_PER_HASH == 16 +__attribute__((reqd_work_group_size(32, 1, 1))) +#else +__attribute__((reqd_work_group_size(16, 1, 1))) +#endif +__kernel void execute_vm(__global void* vm_states, __global void* rounding, __global void* scratchpads, __global const void* dataset_ptr, uint32_t batch_size, uint32_t num_iterations, uint32_t first, uint32_t last) +{ + // 2 hashes per warp, 4 KB shared memory for VM states + __local uint64_t vm_states_local[(VM_STATE_SIZE * 2) / sizeof(uint64_t)]; + + load_buffer(vm_states_local, sizeof(vm_states_local) / sizeof(uint64_t), vm_states); + + barrier(CLK_LOCAL_MEM_FENCE); + + enum { IDX_WIDTH = (WORKERS_PER_HASH == 16) ? 16 : 8 }; + + __local uint64_t* R = vm_states_local + (get_local_id(0) / IDX_WIDTH) * VM_STATE_SIZE / sizeof(uint64_t); + __local double* F = (__local double*)(R + 8); + __local double* E = (__local double*)(R + 16); + + const uint32_t global_index = get_global_id(0); + const int32_t idx = global_index / IDX_WIDTH; + const int32_t sub = global_index % IDX_WIDTH; + + uint32_t ma = ((__local uint32_t*)(R + 16))[0]; + uint32_t mx = ((__local uint32_t*)(R + 16))[1]; + + const uint32_t addressRegisters = ((__local uint32_t*)(R + 16))[2]; + __local const uint64_t* readReg0 = (__local uint64_t*)(((__local uint8_t*)R) + (addressRegisters & 0xff)); + __local const uint64_t* readReg1 = (__local uint64_t*)(((__local uint8_t*)R) + ((addressRegisters >> 8) & 0xff)); + __local const uint32_t* readReg2 = (__local uint32_t*)(((__local uint8_t*)R) + ((addressRegisters >> 16) & 0xff)); + __local const uint32_t* readReg3 = (__local uint32_t*)(((__local uint8_t*)R) + (addressRegisters >> 24)); + + const uint32_t datasetOffset = ((__local uint32_t*)(R + 16))[3]; + __global const uint8_t* dataset = ((__global const uint8_t*)dataset_ptr) + datasetOffset; + + const uint32_t fp_reg_offset = 64 + ((global_index & 1) << 3); + const uint32_t fp_reg_group_A_offset = 192 + ((global_index & 1) << 3); + + __local uint64_t* eMask = R + 18; + + const uint32_t program_length = ((__local uint32_t*)(R + 20))[0]; + uint32_t fprc = ((__global uint32_t*)rounding)[idx]; + + uint32_t spAddr0 = first ? mx : 0; + uint32_t spAddr1 = first ? ma : 0; + + __global uint8_t* scratchpad = ((__global uint8_t*)scratchpads) + idx * (uint64_t)(RANDOMX_SCRATCHPAD_L3 + 64); + + const bool f_group = (sub < 4); + + __local double* fe = f_group ? (F + sub * 2) : (E + (sub - 4) * 2); + __local double* f = F + sub; + __local double* e = E + sub; + + const uint64_t andMask = f_group ? (uint64_t)(-1) : dynamicMantissaMask; + const uint64_t orMask1 = f_group ? 0 : eMask[0]; + const uint64_t orMask2 = f_group ? 0 : eMask[1]; + const uint64_t xexponentMask = (sub & 1) ? eMask[1] : eMask[0]; + + __local uint32_t* imm_buf = (__local uint32_t*)(R + REGISTERS_SIZE / sizeof(uint64_t)); + __local const uint32_t* compiled_program = (__local const uint32_t*)(R + (REGISTERS_SIZE + IMM_BUF_SIZE) / sizeof(uint64_t)); + + const uint32_t workers_mask = ((1 << WORKERS_PER_HASH) - 1) << ((get_local_id(0) / IDX_WIDTH) * IDX_WIDTH); + const uint32_t fp_workers_mask = 3 << (((sub >> 1) << 1) + (get_local_id(0) / IDX_WIDTH) * IDX_WIDTH); + + #pragma unroll(1) + for (int ic = 0; ic < num_iterations; ++ic) + { + __local uint64_t *r; + __global uint64_t *p0, *p1; + if ((WORKERS_PER_HASH <= 8) || (sub < 8)) + { + const uint64_t spMix = *readReg0 ^ *readReg1; + spAddr0 ^= ((const uint32_t*)&spMix)[0]; + spAddr1 ^= ((const uint32_t*)&spMix)[1]; + spAddr0 &= ScratchpadL3Mask64; + spAddr1 &= ScratchpadL3Mask64; + + p0 = (__global uint64_t*)(scratchpad + spAddr0 + sub * 8); + p1 = (__global uint64_t*)(scratchpad + spAddr1 + sub * 8); + + r = R + sub; + *r ^= *p0; + + uint64_t global_mem_data = *p1; + int32_t* q = (int32_t*)&global_mem_data; + + fe[0] = load_F_E_groups(q[0], andMask, orMask1); + fe[1] = load_F_E_groups(q[1], andMask, orMask2); + } + + //if ((global_index == 0) && (ic == 0)) + //{ + // printf("ic = %d (before)\n", ic); + // for (int i = 0; i < 8; ++i) + // printf("f%d = %016llx\n", i, bit_cast(F[i])); + // for (int i = 0; i < 8; ++i) + // printf("e%d = %016llx\n", i, bit_cast(E[i])); + // printf("\n"); + //} + + if ((WORKERS_PER_HASH == IDX_WIDTH) || (sub < WORKERS_PER_HASH)) + fprc = inner_loop(program_length, compiled_program, sub, scratchpad, fp_reg_offset, fp_reg_group_A_offset, R, imm_buf, batch_size, fprc, fp_workers_mask, xexponentMask, workers_mask); + + //if ((global_index == 0) && (ic == RANDOMX_PROGRAM_ITERATIONS - 1)) + //{ + // printf("ic = %d (after)\n", ic); + // for (int i = 0; i < 8; ++i) + // printf("r%d = %016llx\n", i, R[i]); + // for (int i = 0; i < 8; ++i) + // printf("f%d = %016llx\n", i, bit_cast(F[i])); + // for (int i = 0; i < 8; ++i) + // printf("e%d = %016llx\n", i, bit_cast(E[i])); + // printf("\n"); + //} + + if ((WORKERS_PER_HASH <= 8) || (sub < 8)) + { + mx ^= *readReg2 ^ *readReg3; + mx &= CacheLineAlignMask; + + const uint64_t next_r = *r ^ *(__global const uint64_t*)(dataset + ma + sub * 8); + *r = next_r; + + *p1 = next_r; + *p0 = as_ulong(f[0]) ^ as_ulong(e[0]); + + uint32_t tmp = ma; + ma = mx; + mx = tmp; + + spAddr0 = 0; + spAddr1 = 0; + } + } + + //if (global_index == 0) + //{ + // for (int i = 0; i < 8; ++i) + // printf("r%d = %016llx\n", i, R[i]); + // for (int i = 0; i < 8; ++i) + // printf("fe%d = %016llx\n", i, bit_cast(F[i]) ^ bit_cast(E[i])); + // printf("\n"); + //} + + if ((WORKERS_PER_HASH > 8) && (sub >= 8)) + return; + + __global uint64_t* p = ((__global uint64_t*)vm_states) + idx * (VM_STATE_SIZE / sizeof(uint64_t)); + p[sub] = R[sub]; + + if (sub == 0) + { + ((__global uint32_t*)rounding)[idx] = fprc; + } + + if (last) + { + p[sub + 8] = as_ulong(F[sub]) ^ as_ulong(E[sub]); + p[sub + 16] = as_ulong(E[sub]); + } + else if (sub == 0) + { + ((__global uint32_t*)(p + 16))[0] = ma; + ((__global uint32_t*)(p + 16))[1] = mx; + } +} + +__attribute__((reqd_work_group_size(64, 1, 1))) +__kernel void find_shares(__global const uint64_t* hashes, uint64_t target, uint32_t start_nonce, __global uint32_t* shares) +{ + const uint32_t global_index = get_global_id(0); + + if (hashes[global_index * 4 + 3] < target) { + //if (global_index == 0) { + const uint32_t idx = atomic_inc(shares + 0xFF); + if (idx < 0xFF) { + shares[idx] = start_nonce + global_index; + } + } +} + +)===" diff --git a/src/backend/opencl/opencl.cmake b/src/backend/opencl/opencl.cmake index 2130a14d..8b68d2c3 100644 --- a/src/backend/opencl/opencl.cmake +++ b/src/backend/opencl/opencl.cmake @@ -4,6 +4,7 @@ if (WITH_OPENCL) add_definitions(/DXMRIG_FEATURE_OPENCL) set(HEADERS_BACKEND_OPENCL + src/backend/opencl/cl/OclSource.h src/backend/opencl/OclBackend.h src/backend/opencl/OclCache.h src/backend/opencl/OclConfig.h @@ -20,6 +21,7 @@ if (WITH_OPENCL) ) set(SOURCES_BACKEND_OPENCL + src/backend/opencl/cl/OclSource.cpp src/backend/opencl/OclBackend.cpp src/backend/opencl/OclCache.cpp src/backend/opencl/OclConfig.cpp diff --git a/src/crypto/rx/Rx.cpp b/src/crypto/rx/Rx.cpp index 2b8d06e6..2f9cb4d2 100644 --- a/src/crypto/rx/Rx.cpp +++ b/src/crypto/rx/Rx.cpp @@ -182,7 +182,7 @@ public: d_ptr->getOrAllocate(nodeId)->init(d_ptr->seed(), threads); d_ptr->asyncSend(); - LOG_INFO("%s" CYAN_BOLD("#%u") GREEN(" init done") BLACK_BOLD(" (%" PRIu64 " ms)"), tag, nodeId, Chrono::steadyMSecs() - ts); + LOG_INFO("%s" CYAN_BOLD("#%u") GREEN(" init done ") CYAN_BOLD("%zu/%zu") BLACK_BOLD(" (%" PRIu64 " ms)"), tag, nodeId, d_ptr->m_ready, d_ptr->count(), Chrono::steadyMSecs() - ts); }