From df559b44d872837e119cfa8367f1ed2b519bb7a7 Mon Sep 17 00:00:00 2001 From: XMRig Date: Sat, 3 Aug 2019 23:08:57 +0700 Subject: [PATCH 01/26] v2.99.5-evo --- src/version.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/version.h b/src/version.h index 1e1ca8b6..e2c62435 100644 --- a/src/version.h +++ b/src/version.h @@ -28,7 +28,7 @@ #define APP_ID "xmrig" #define APP_NAME "XMRig" #define APP_DESC "XMRig CPU miner" -#define APP_VERSION "2.99.4-beta" +#define APP_VERSION "2.99.5-evo" #define APP_DOMAIN "xmrig.com" #define APP_SITE "www.xmrig.com" #define APP_COPYRIGHT "Copyright (C) 2016-2019 xmrig.com" @@ -36,7 +36,7 @@ #define APP_VER_MAJOR 2 #define APP_VER_MINOR 99 -#define APP_VER_PATCH 4 +#define APP_VER_PATCH 5 #ifdef _MSC_VER # if (_MSC_VER >= 1920) From 7eaf7764f73e68af5965a766a18b13fa2f1f5d1e Mon Sep 17 00:00:00 2001 From: XMRig Date: Sun, 4 Aug 2019 19:47:23 +0700 Subject: [PATCH 02/26] rapidjson updated to recent git version. --- src/3rdparty/rapidjson/allocators.h | 17 +- src/3rdparty/rapidjson/cursorstreamwrapper.h | 78 ++ src/3rdparty/rapidjson/document.h | 265 ++++--- src/3rdparty/rapidjson/encodedstream.h | 2 +- src/3rdparty/rapidjson/encodings.h | 86 +-- src/3rdparty/rapidjson/error/error.h | 10 +- src/3rdparty/rapidjson/filereadstream.h | 4 +- src/3rdparty/rapidjson/filewritestream.h | 4 +- src/3rdparty/rapidjson/internal/biginteger.h | 4 +- src/3rdparty/rapidjson/internal/diyfp.h | 37 +- src/3rdparty/rapidjson/internal/dtoa.h | 8 +- src/3rdparty/rapidjson/internal/ieee754.h | 4 +- src/3rdparty/rapidjson/internal/itoa.h | 82 ++- src/3rdparty/rapidjson/internal/meta.h | 9 +- src/3rdparty/rapidjson/internal/regex.h | 319 +++++---- src/3rdparty/rapidjson/internal/stack.h | 10 +- src/3rdparty/rapidjson/internal/strfunc.h | 14 + src/3rdparty/rapidjson/internal/strtod.h | 111 +-- src/3rdparty/rapidjson/istreamwrapper.h | 87 ++- src/3rdparty/rapidjson/license.txt | 57 ++ src/3rdparty/rapidjson/pointer.h | 92 ++- src/3rdparty/rapidjson/prettywriter.h | 60 +- src/3rdparty/rapidjson/rapidjson.h | 112 ++- src/3rdparty/rapidjson/reader.h | 601 ++++++++++++---- src/3rdparty/rapidjson/schema.h | 717 ++++++++++++++++--- src/3rdparty/rapidjson/stream.h | 52 +- src/3rdparty/rapidjson/stringbuffer.h | 4 + src/3rdparty/rapidjson/writer.h | 159 +++- 28 files changed, 2228 insertions(+), 777 deletions(-) create mode 100644 src/3rdparty/rapidjson/cursorstreamwrapper.h create mode 100644 src/3rdparty/rapidjson/license.txt diff --git a/src/3rdparty/rapidjson/allocators.h b/src/3rdparty/rapidjson/allocators.h index 98affe03..cc67c897 100644 --- a/src/3rdparty/rapidjson/allocators.h +++ b/src/3rdparty/rapidjson/allocators.h @@ -52,6 +52,19 @@ concept Allocator { \endcode */ + +/*! \def RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY + \ingroup RAPIDJSON_CONFIG + \brief User-defined kDefaultChunkCapacity definition. + + User can define this as any \c size that is a power of 2. +*/ + +#ifndef RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY +#define RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY (64 * 1024) +#endif + + /////////////////////////////////////////////////////////////////////////////// // CrtAllocator @@ -236,7 +249,7 @@ private: */ bool AddChunk(size_t capacity) { if (!baseAllocator_) - ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator()); + ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)(); if (ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) { chunk->capacity = capacity; chunk->size = 0; @@ -248,7 +261,7 @@ private: return false; } - static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. + static const int kDefaultChunkCapacity = RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY; //!< Default chunk capacity. //! Chunk header for perpending to each chunk. /*! Chunks are stored as a singly linked list. diff --git a/src/3rdparty/rapidjson/cursorstreamwrapper.h b/src/3rdparty/rapidjson/cursorstreamwrapper.h new file mode 100644 index 00000000..52c11a7c --- /dev/null +++ b/src/3rdparty/rapidjson/cursorstreamwrapper.h @@ -0,0 +1,78 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_CURSORSTREAMWRAPPER_H_ +#define RAPIDJSON_CURSORSTREAMWRAPPER_H_ + +#include "stream.h" + +#if defined(__GNUC__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#if defined(_MSC_VER) && _MSC_VER <= 1800 +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4702) // unreachable code +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +RAPIDJSON_NAMESPACE_BEGIN + + +//! Cursor stream wrapper for counting line and column number if error exists. +/*! + \tparam InputStream Any stream that implements Stream Concept +*/ +template > +class CursorStreamWrapper : public GenericStreamWrapper { +public: + typedef typename Encoding::Ch Ch; + + CursorStreamWrapper(InputStream& is): + GenericStreamWrapper(is), line_(1), col_(0) {} + + // counting line and column number + Ch Take() { + Ch ch = this->is_.Take(); + if(ch == '\n') { + line_ ++; + col_ = 0; + } else { + col_ ++; + } + return ch; + } + + //! Get the error line number, if error exists. + size_t GetLine() const { return line_; } + //! Get the error column number, if error exists. + size_t GetColumn() const { return col_; } + +private: + size_t line_; //!< Current Line + size_t col_; //!< Current Column +}; + +#if defined(_MSC_VER) && _MSC_VER <= 1800 +RAPIDJSON_DIAG_POP +#endif + +#if defined(__GNUC__) +RAPIDJSON_DIAG_POP +#endif + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_CURSORSTREAMWRAPPER_H_ diff --git a/src/3rdparty/rapidjson/document.h b/src/3rdparty/rapidjson/document.h index e3e20dfb..9783fe4a 100644 --- a/src/3rdparty/rapidjson/document.h +++ b/src/3rdparty/rapidjson/document.h @@ -26,26 +26,21 @@ #include RAPIDJSON_DIAG_PUSH -#ifdef _MSC_VER -RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant -RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible loss of data -#endif - #ifdef __clang__ RAPIDJSON_DIAG_OFF(padded) RAPIDJSON_DIAG_OFF(switch-enum) RAPIDJSON_DIAG_OFF(c++98-compat) +#elif defined(_MSC_VER) +RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible loss of data #endif #ifdef __GNUC__ RAPIDJSON_DIAG_OFF(effc++) -#if __GNUC__ >= 6 -RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_NOEXCEPT functions -#endif #endif // __GNUC__ #ifndef RAPIDJSON_NOMEMBERITERATORCLASS -#include // std::iterator, std::random_access_iterator_tag +#include // std::random_access_iterator_tag #endif #if RAPIDJSON_HAS_CXX11_RVALUE_REFS @@ -71,6 +66,12 @@ template struct GenericMember { GenericValue name; //!< name of member (must be a string) GenericValue value; //!< value of member. + + // swap() for std::sort() and other potential use in STL. + friend inline void swap(GenericMember& a, GenericMember& b) RAPIDJSON_NOEXCEPT { + a.name.Swap(b.name); + a.value.Swap(b.value); + } }; /////////////////////////////////////////////////////////////////////////////// @@ -98,16 +99,13 @@ struct GenericMember { \see GenericMember, GenericValue::MemberIterator, GenericValue::ConstMemberIterator */ template -class GenericMemberIterator - : public std::iterator >::Type> { +class GenericMemberIterator { friend class GenericValue; template friend class GenericMemberIterator; typedef GenericMember PlainType; typedef typename internal::MaybeAddConst::Type ValueType; - typedef std::iterator BaseType; public: //! Iterator type itself @@ -117,12 +115,21 @@ public: //! Non-constant iterator type typedef GenericMemberIterator NonConstIterator; + /** \name std::iterator_traits support */ + //@{ + typedef ValueType value_type; + typedef ValueType * pointer; + typedef ValueType & reference; + typedef std::ptrdiff_t difference_type; + typedef std::random_access_iterator_tag iterator_category; + //@} + //! Pointer to (const) GenericMember - typedef typename BaseType::pointer Pointer; + typedef pointer Pointer; //! Reference to (const) GenericMember - typedef typename BaseType::reference Reference; + typedef reference Reference; //! Signed integer type (e.g. \c ptrdiff_t) - typedef typename BaseType::difference_type DifferenceType; + typedef difference_type DifferenceType; //! Default constructor (singular value) /*! Creates an iterator pointing to no element. @@ -198,17 +205,17 @@ private: // class-based member iterator implementation disabled, use plain pointers template -struct GenericMemberIterator; +class GenericMemberIterator; //! non-const GenericMemberIterator template -struct GenericMemberIterator { +class GenericMemberIterator { //! use plain pointer as iterator type typedef GenericMember* Iterator; }; //! const GenericMemberIterator template -struct GenericMemberIterator { +class GenericMemberIterator { //! use plain const pointer as iterator type typedef const GenericMember* Iterator; }; @@ -300,7 +307,7 @@ struct GenericStringRef { */ #endif explicit GenericStringRef(const CharType* str) - : s(str), length(internal::StrLen(str)){ RAPIDJSON_ASSERT(s != 0); } + : s(str), length(NotNullStrLen(str)) {} //! Create constant string reference from pointer and length #ifndef __clang__ // -Wdocumentation @@ -312,12 +319,10 @@ struct GenericStringRef { */ #endif GenericStringRef(const CharType* str, SizeType len) - : s(str), length(len) { RAPIDJSON_ASSERT(s != 0); } + : s(RAPIDJSON_LIKELY(str) ? str : emptyString), length(len) { RAPIDJSON_ASSERT(str != 0 || len == 0u); } GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {} - GenericStringRef& operator=(const GenericStringRef& rhs) { s = rhs.s; length = rhs.length; } - //! implicit conversion to plain CharType pointer operator const Ch *() const { return s; } @@ -325,11 +330,24 @@ struct GenericStringRef { const SizeType length; //!< length of the string (excluding the trailing NULL terminator) private: + SizeType NotNullStrLen(const CharType* str) { + RAPIDJSON_ASSERT(str != 0); + return internal::StrLen(str); + } + + /// Empty string - used when passing in a NULL pointer + static const Ch emptyString[]; + //! Disallow construction from non-const array template GenericStringRef(CharType (&str)[N]) /* = delete */; + //! Copy assignment operator not permitted - immutable type + GenericStringRef& operator=(const GenericStringRef& rhs) /* = delete */; }; +template +const CharType GenericStringRef::emptyString[] = { CharType() }; + //! Mark a character pointer as constant string /*! Mark a plain character pointer as a "string literal". This function can be used to avoid copying a character string to be referenced as a @@ -344,7 +362,7 @@ private: */ template inline GenericStringRef StringRef(const CharType* str) { - return GenericStringRef(str, internal::StrLen(str)); + return GenericStringRef(str); } //! Mark a character pointer as constant string @@ -434,6 +452,26 @@ struct TypeHelper { static ValueType& Set(ValueType& v, unsigned data, typename ValueType::AllocatorType&) { return v.SetUint(data); } }; +#ifdef _MSC_VER +RAPIDJSON_STATIC_ASSERT(sizeof(long) == sizeof(int)); +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsInt(); } + static long Get(const ValueType& v) { return v.GetInt(); } + static ValueType& Set(ValueType& v, long data) { return v.SetInt(data); } + static ValueType& Set(ValueType& v, long data, typename ValueType::AllocatorType&) { return v.SetInt(data); } +}; + +RAPIDJSON_STATIC_ASSERT(sizeof(unsigned long) == sizeof(unsigned)); +template +struct TypeHelper { + static bool Is(const ValueType& v) { return v.IsUint(); } + static unsigned long Get(const ValueType& v) { return v.GetUint(); } + static ValueType& Set(ValueType& v, unsigned long data) { return v.SetUint(data); } + static ValueType& Set(ValueType& v, unsigned long data, typename ValueType::AllocatorType&) { return v.SetUint(data); } +}; +#endif + template struct TypeHelper { static bool Is(const ValueType& v) { return v.IsInt64(); } @@ -507,7 +545,7 @@ struct TypeHelper { static bool Is(const ValueType& v) { return v.IsObject(); } static ObjectType Get(ValueType& v) { return v.GetObject(); } static ValueType& Set(ValueType& v, ObjectType data) { return v = data; } - static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { v = data; } + static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { return v = data; } }; template @@ -590,11 +628,11 @@ public: \note Default content for number is zero. */ explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_() { - static const uint16_t defaultFlags[7] = { + static const uint16_t defaultFlags[] = { kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kShortStringFlag, kNumberAnyFlag }; - RAPIDJSON_ASSERT(type <= kNumberType); + RAPIDJSON_NOEXCEPT_ASSERT(type >= kNullType && type <= kNumberType); data_.f.flags = defaultFlags[type]; // Use ShortString to store empty string. @@ -607,10 +645,50 @@ public: \tparam SourceAllocator allocator of \c rhs \param rhs Value to copy from (read-only) \param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator(). + \param copyConstStrings Force copying of constant strings (e.g. referencing an in-situ buffer) \see CopyFrom() */ - template< typename SourceAllocator > - GenericValue(const GenericValue& rhs, Allocator & allocator); + template + GenericValue(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { + switch (rhs.GetType()) { + case kObjectType: { + SizeType count = rhs.data_.o.size; + Member* lm = reinterpret_cast(allocator.Malloc(count * sizeof(Member))); + const typename GenericValue::Member* rm = rhs.GetMembersPointer(); + for (SizeType i = 0; i < count; i++) { + new (&lm[i].name) GenericValue(rm[i].name, allocator, copyConstStrings); + new (&lm[i].value) GenericValue(rm[i].value, allocator, copyConstStrings); + } + data_.f.flags = kObjectFlag; + data_.o.size = data_.o.capacity = count; + SetMembersPointer(lm); + } + break; + case kArrayType: { + SizeType count = rhs.data_.a.size; + GenericValue* le = reinterpret_cast(allocator.Malloc(count * sizeof(GenericValue))); + const GenericValue* re = rhs.GetElementsPointer(); + for (SizeType i = 0; i < count; i++) + new (&le[i]) GenericValue(re[i], allocator, copyConstStrings); + data_.f.flags = kArrayFlag; + data_.a.size = data_.a.capacity = count; + SetElementsPointer(le); + } + break; + case kStringType: + if (rhs.data_.f.flags == kConstStringFlag && !copyConstStrings) { + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + } + else + SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); + break; + default: + data_.f.flags = rhs.data_.f.flags; + data_ = *reinterpret_cast(&rhs.data_); + break; + } + } //! Constructor for boolean value. /*! \param b Boolean value @@ -672,6 +750,9 @@ public: //! Constructor for double value. explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; } + //! Constructor for float value. + explicit GenericValue(float f) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = static_cast(f); data_.f.flags = kNumberDoubleFlag; } + //! Constructor for constant string (i.e. do not make a copy of string) GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); } @@ -753,9 +834,10 @@ public: /*! \param rhs Source of the assignment. It will become a null value after assignment. */ GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT { - RAPIDJSON_ASSERT(this != &rhs); - this->~GenericValue(); - RawAssign(rhs); + if (RAPIDJSON_LIKELY(this != &rhs)) { + this->~GenericValue(); + RawAssign(rhs); + } return *this; } @@ -800,12 +882,13 @@ public: \tparam SourceAllocator Allocator type of \c rhs \param rhs Value to copy from (read-only) \param allocator Allocator to use for copying + \param copyConstStrings Force copying of constant strings (e.g. referencing an in-situ buffer) */ template - GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator) { + GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { RAPIDJSON_ASSERT(static_cast(this) != static_cast(&rhs)); this->~GenericValue(); - new (this) GenericValue(rhs, allocator); + new (this) GenericValue(rhs, allocator, copyConstStrings); return *this; } @@ -846,7 +929,7 @@ public: //! Equal-to operator /*! \note If an object contains duplicated named member, comparing equality with any object is always \c false. - \note Linear time complexity (number of all values in the subtree and total lengths of all strings). + \note Complexity is quadratic in Object's member number and linear for the rest (number of all values in the subtree and total lengths of all strings). */ template bool operator==(const GenericValue& rhs) const { @@ -955,14 +1038,14 @@ public: uint64_t u = GetUint64(); volatile double d = static_cast(u); return (d >= 0.0) - && (d < static_cast(std::numeric_limits::max())) + && (d < static_cast((std::numeric_limits::max)())) && (u == static_cast(d)); } if (IsInt64()) { int64_t i = GetInt64(); volatile double d = static_cast(i); - return (d >= static_cast(std::numeric_limits::min())) - && (d < static_cast(std::numeric_limits::max())) + return (d >= static_cast((std::numeric_limits::min)())) + && (d < static_cast((std::numeric_limits::max)())) && (i == static_cast(d)); } return true; // double, int, uint are always lossless @@ -979,8 +1062,8 @@ public: bool IsLosslessFloat() const { if (!IsNumber()) return false; double a = GetDouble(); - if (a < static_cast(-std::numeric_limits::max()) - || a > static_cast(std::numeric_limits::max())) + if (a < static_cast(-(std::numeric_limits::max)()) + || a > static_cast((std::numeric_limits::max)())) return false; double b = static_cast(static_cast(a)); return a >= b && a <= b; // Prevent -Wfloat-equal @@ -1015,6 +1098,9 @@ public: //! Get the number of members in the object. SizeType MemberCount() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size; } + //! Get the capacity of object. + SizeType MemberCapacity() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.capacity; } + //! Check whether the object is empty. bool ObjectEmpty() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size == 0; } @@ -1083,6 +1169,21 @@ public: /*! \pre IsObject() == true */ MemberIterator MemberEnd() { RAPIDJSON_ASSERT(IsObject()); return MemberIterator(GetMembersPointer() + data_.o.size); } + //! Request the object to have enough capacity to store members. + /*! \param newCapacity The capacity that the object at least need to have. + \param allocator Allocator for reallocating memory. It must be the same one as used before. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \note Linear time complexity. + */ + GenericValue& MemberReserve(SizeType newCapacity, Allocator &allocator) { + RAPIDJSON_ASSERT(IsObject()); + if (newCapacity > data_.o.capacity) { + SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), data_.o.capacity * sizeof(Member), newCapacity * sizeof(Member)))); + data_.o.capacity = newCapacity; + } + return *this; + } + //! Check whether a member exists in the object. /*! \param name Member name to be searched. @@ -1188,17 +1289,8 @@ public: RAPIDJSON_ASSERT(name.IsString()); ObjectData& o = data_.o; - if (o.size >= o.capacity) { - if (o.capacity == 0) { - o.capacity = kDefaultObjectCapacity; - SetMembersPointer(reinterpret_cast(allocator.Malloc(o.capacity * sizeof(Member)))); - } - else { - SizeType oldCapacity = o.capacity; - o.capacity += (oldCapacity + 1) / 2; // grow by factor 1.5 - SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), oldCapacity * sizeof(Member), o.capacity * sizeof(Member)))); - } - } + if (o.size >= o.capacity) + MemberReserve(o.capacity == 0 ? kDefaultObjectCapacity : (o.capacity + (o.capacity + 1) / 2), allocator); Member* members = GetMembersPointer(); members[o.size].name.RawAssign(name); members[o.size].value.RawAssign(value); @@ -1425,7 +1517,7 @@ public: MemberIterator pos = MemberBegin() + (first - MemberBegin()); for (MemberIterator itr = pos; itr != last; ++itr) itr->~Member(); - std::memmove(&*pos, &*last, static_cast(MemberEnd() - last) * sizeof(Member)); + std::memmove(static_cast(&*pos), &*last, static_cast(MemberEnd() - last) * sizeof(Member)); data_.o.size -= static_cast(last - first); return pos; } @@ -1628,8 +1720,8 @@ public: RAPIDJSON_ASSERT(last <= End()); ValueIterator pos = Begin() + (first - Begin()); for (ValueIterator itr = pos; itr != last; ++itr) - itr->~GenericValue(); - std::memmove(pos, last, static_cast(End() - last) * sizeof(GenericValue)); + itr->~GenericValue(); + std::memmove(static_cast(pos), last, static_cast(End() - last) * sizeof(GenericValue)); data_.a.size -= static_cast(last - first); return pos; } @@ -1671,7 +1763,7 @@ public: GenericValue& SetInt64(int64_t i64) { this->~GenericValue(); new (this) GenericValue(i64); return *this; } GenericValue& SetUint64(uint64_t u64) { this->~GenericValue(); new (this) GenericValue(u64); return *this; } GenericValue& SetDouble(double d) { this->~GenericValue(); new (this) GenericValue(d); return *this; } - GenericValue& SetFloat(float f) { this->~GenericValue(); new (this) GenericValue(f); return *this; } + GenericValue& SetFloat(float f) { this->~GenericValue(); new (this) GenericValue(static_cast(f)); return *this; } //@} @@ -1710,7 +1802,7 @@ public: \return The value itself for fluent API. \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length */ - GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { this->~GenericValue(); SetStringRaw(StringRef(s, length), allocator); return *this; } + GenericValue& SetString(const Ch* s, SizeType length, Allocator& allocator) { return SetString(StringRef(s, length), allocator); } //! Set this value as a string by copying from source string. /*! \param s source string. @@ -1718,7 +1810,15 @@ public: \return The value itself for fluent API. \post IsString() == true && GetString() != s && strcmp(GetString(),s) == 0 && GetStringLength() == length */ - GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(s, internal::StrLen(s), allocator); } + GenericValue& SetString(const Ch* s, Allocator& allocator) { return SetString(StringRef(s), allocator); } + + //! Set this value as a string by copying from source string. + /*! \param s source string reference + \param allocator Allocator for allocating copied buffer. Commonly use GenericDocument::GetAllocator(). + \return The value itself for fluent API. + \post IsString() == true && GetString() != s.s && strcmp(GetString(),s) == 0 && GetStringLength() == length + */ + GenericValue& SetString(StringRefType s, Allocator& allocator) { this->~GenericValue(); SetStringRaw(s, allocator); return *this; } #if RAPIDJSON_HAS_STDSTRING //! Set this value as a string by copying from source string. @@ -1728,7 +1828,7 @@ public: \post IsString() == true && GetString() != s.data() && strcmp(GetString(),s.data() == 0 && GetStringLength() == s.size() \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. */ - GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(s.data(), SizeType(s.size()), allocator); } + GenericValue& SetString(const std::basic_string& s, Allocator& allocator) { return SetString(StringRef(s), allocator); } #endif //@} @@ -1936,7 +2036,7 @@ private: if (count) { GenericValue* e = static_cast(allocator.Malloc(count * sizeof(GenericValue))); SetElementsPointer(e); - std::memcpy(e, values, count * sizeof(GenericValue)); + std::memcpy(static_cast(e), values, count * sizeof(GenericValue)); } else SetElementsPointer(0); @@ -1949,7 +2049,7 @@ private: if (count) { Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); SetMembersPointer(m); - std::memcpy(m, members, count * sizeof(Member)); + std::memcpy(static_cast(m), members, count * sizeof(Member)); } else SetMembersPointer(0); @@ -2038,7 +2138,7 @@ public: GenericValue(type), allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() { if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); } //! Constructor @@ -2051,7 +2151,7 @@ public: allocator_(allocator), ownAllocator_(0), stack_(stackAllocator, stackCapacity), parseResult_() { if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); } #if RAPIDJSON_HAS_CXX11_RVALUE_REFS @@ -2112,6 +2212,10 @@ public: return *this; } + // Allow Swap with ValueType. + // Refer to Effective C++ 3rd Edition/Item 33: Avoid hiding inherited names. + using ValueType::Swap; + //! free-standing swap function helper /*! Helper function to enable support for common swap implementation pattern based on \c std::swap: @@ -2243,7 +2347,7 @@ public: template GenericDocument& Parse(const typename SourceEncoding::Ch* str, size_t length) { RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); - MemoryStream ms(static_cast(str), length * sizeof(typename SourceEncoding::Ch)); + MemoryStream ms(reinterpret_cast(str), length * sizeof(typename SourceEncoding::Ch)); EncodedInputStream is(ms); ParseStream(is); return *this; @@ -2280,7 +2384,7 @@ public: //!@name Handling parse errors //!@{ - //! Whether a parse error has occured in the last parsing. + //! Whether a parse error has occurred in the last parsing. bool HasParseError() const { return parseResult_.IsError(); } //! Get the \ref ParseErrorCode of last parsing. @@ -2401,35 +2505,6 @@ private: //! GenericDocument with UTF8 encoding typedef GenericDocument > Document; -// defined here due to the dependency on GenericDocument -template -template -inline -GenericValue::GenericValue(const GenericValue& rhs, Allocator& allocator) -{ - switch (rhs.GetType()) { - case kObjectType: - case kArrayType: { // perform deep copy via SAX Handler - GenericDocument d(&allocator); - rhs.Accept(d); - RawAssign(*d.stack_.template Pop(1)); - } - break; - case kStringType: - if (rhs.data_.f.flags == kConstStringFlag) { - data_.f.flags = rhs.data_.f.flags; - data_ = *reinterpret_cast(&rhs.data_); - } else { - SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator); - } - break; - default: - data_.f.flags = rhs.data_.f.flags; - data_ = *reinterpret_cast(&rhs.data_); - break; - } -} - //! Helper class for accessing Value of array type. /*! Instance of this helper class is obtained by \c GenericValue::GetArray(). @@ -2510,6 +2585,7 @@ public: ~GenericObject() {} SizeType MemberCount() const { return value_.MemberCount(); } + SizeType MemberCapacity() const { return value_.MemberCapacity(); } bool ObjectEmpty() const { return value_.ObjectEmpty(); } template ValueType& operator[](T* name) const { return value_[name]; } template ValueType& operator[](const GenericValue& name) const { return value_[name]; } @@ -2518,6 +2594,7 @@ public: #endif MemberIterator MemberBegin() const { return value_.MemberBegin(); } MemberIterator MemberEnd() const { return value_.MemberEnd(); } + GenericObject MemberReserve(SizeType newCapacity, AllocatorType &allocator) const { value_.MemberReserve(newCapacity, allocator); return *this; } bool HasMember(const Ch* name) const { return value_.HasMember(name); } #if RAPIDJSON_HAS_STDSTRING bool HasMember(const std::basic_string& name) const { return value_.HasMember(name); } @@ -2543,7 +2620,7 @@ public: GenericObject AddMember(StringRefType name, ValueType& value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } GenericObject AddMember(StringRefType name, StringRefType value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (GenericObject)) AddMember(StringRefType name, T value, AllocatorType& allocator) const { value_.AddMember(name, value, allocator); return *this; } - void RemoveAllMembers() { return value_.RemoveAllMembers(); } + void RemoveAllMembers() { value_.RemoveAllMembers(); } bool RemoveMember(const Ch* name) const { return value_.RemoveMember(name); } #if RAPIDJSON_HAS_STDSTRING bool RemoveMember(const std::basic_string& name) const { return value_.RemoveMember(name); } diff --git a/src/3rdparty/rapidjson/encodedstream.h b/src/3rdparty/rapidjson/encodedstream.h index 14506838..223601c0 100644 --- a/src/3rdparty/rapidjson/encodedstream.h +++ b/src/3rdparty/rapidjson/encodedstream.h @@ -200,7 +200,7 @@ private: // xx xx xx xx UTF-8 if (!hasBOM_) { - unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); + int pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); switch (pattern) { case 0x08: type_ = kUTF32BE; break; case 0x0A: type_ = kUTF16BE; break; diff --git a/src/3rdparty/rapidjson/encodings.h b/src/3rdparty/rapidjson/encodings.h index baa7c2b1..0b244679 100644 --- a/src/3rdparty/rapidjson/encodings.h +++ b/src/3rdparty/rapidjson/encodings.h @@ -17,7 +17,7 @@ #include "rapidjson.h" -#ifdef _MSC_VER +#if defined(_MSC_VER) && !defined(__clang__) RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4244) // conversion from 'type1' to 'type2', possible loss of data RAPIDJSON_DIAG_OFF(4702) // unreachable code @@ -144,9 +144,9 @@ struct UTF8 { template static bool Decode(InputStream& is, unsigned* codepoint) { -#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast(c) & 0x3Fu) -#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) -#define TAIL() COPY(); TRANS(0x70) +#define RAPIDJSON_COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast(c) & 0x3Fu) +#define RAPIDJSON_TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) +#define RAPIDJSON_TAIL() RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x70) typename InputStream::Ch c = is.Take(); if (!(c & 0x80)) { *codepoint = static_cast(c); @@ -157,48 +157,48 @@ struct UTF8 { if (type >= 32) { *codepoint = 0; } else { - *codepoint = (0xFF >> type) & static_cast(c); + *codepoint = (0xFFu >> type) & static_cast(c); } bool result = true; switch (type) { - case 2: TAIL(); return result; - case 3: TAIL(); TAIL(); return result; - case 4: COPY(); TRANS(0x50); TAIL(); return result; - case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; - case 6: TAIL(); TAIL(); TAIL(); return result; - case 10: COPY(); TRANS(0x20); TAIL(); return result; - case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; + case 2: RAPIDJSON_TAIL(); return result; + case 3: RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result; + case 4: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x50); RAPIDJSON_TAIL(); return result; + case 5: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x10); RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result; + case 6: RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result; + case 10: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x20); RAPIDJSON_TAIL(); return result; + case 11: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x60); RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result; default: return false; } -#undef COPY -#undef TRANS -#undef TAIL +#undef RAPIDJSON_COPY +#undef RAPIDJSON_TRANS +#undef RAPIDJSON_TAIL } template static bool Validate(InputStream& is, OutputStream& os) { -#define COPY() os.Put(c = is.Take()) -#define TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) -#define TAIL() COPY(); TRANS(0x70) +#define RAPIDJSON_COPY() os.Put(c = is.Take()) +#define RAPIDJSON_TRANS(mask) result &= ((GetRange(static_cast(c)) & mask) != 0) +#define RAPIDJSON_TAIL() RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x70) Ch c; - COPY(); + RAPIDJSON_COPY(); if (!(c & 0x80)) return true; bool result = true; switch (GetRange(static_cast(c))) { - case 2: TAIL(); return result; - case 3: TAIL(); TAIL(); return result; - case 4: COPY(); TRANS(0x50); TAIL(); return result; - case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; - case 6: TAIL(); TAIL(); TAIL(); return result; - case 10: COPY(); TRANS(0x20); TAIL(); return result; - case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; + case 2: RAPIDJSON_TAIL(); return result; + case 3: RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result; + case 4: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x50); RAPIDJSON_TAIL(); return result; + case 5: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x10); RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result; + case 6: RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result; + case 10: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x20); RAPIDJSON_TAIL(); return result; + case 11: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x60); RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result; default: return false; } -#undef COPY -#undef TRANS -#undef TAIL +#undef RAPIDJSON_COPY +#undef RAPIDJSON_TRANS +#undef RAPIDJSON_TAIL } static unsigned char GetRange(unsigned char c) { @@ -283,7 +283,7 @@ struct UTF16 { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); unsigned v = codepoint - 0x10000; os.Put(static_cast((v >> 10) | 0xD800)); - os.Put((v & 0x3FF) | 0xDC00); + os.Put(static_cast((v & 0x3FF) | 0xDC00)); } } @@ -299,7 +299,7 @@ struct UTF16 { RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); unsigned v = codepoint - 0x10000; PutUnsafe(os, static_cast((v >> 10) | 0xD800)); - PutUnsafe(os, (v & 0x3FF) | 0xDC00); + PutUnsafe(os, static_cast((v & 0x3FF) | 0xDC00)); } } @@ -384,7 +384,7 @@ struct UTF16BE : UTF16 { static CharType Take(InputByteStream& is) { RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); unsigned c = static_cast(static_cast(is.Take())) << 8; - c |= static_cast(is.Take()); + c |= static_cast(static_cast(is.Take())); return static_cast(c); } @@ -620,28 +620,28 @@ struct AutoUTF { #define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x template - RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) { + static RAPIDJSON_FORCEINLINE void Encode(OutputStream& os, unsigned codepoint) { typedef void (*EncodeFunc)(OutputStream&, unsigned); static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; (*f[os.GetType()])(os, codepoint); } template - RAPIDJSON_FORCEINLINE static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { + static RAPIDJSON_FORCEINLINE void EncodeUnsafe(OutputStream& os, unsigned codepoint) { typedef void (*EncodeFunc)(OutputStream&, unsigned); static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) }; (*f[os.GetType()])(os, codepoint); } template - RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) { + static RAPIDJSON_FORCEINLINE bool Decode(InputStream& is, unsigned* codepoint) { typedef bool (*DecodeFunc)(InputStream&, unsigned*); static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; return (*f[is.GetType()])(is, codepoint); } template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { typedef bool (*ValidateFunc)(InputStream&, OutputStream&); static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; return (*f[is.GetType()])(is, os); @@ -658,7 +658,7 @@ template struct Transcoder { //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. template - RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Transcode(InputStream& is, OutputStream& os) { unsigned codepoint; if (!SourceEncoding::Decode(is, &codepoint)) return false; @@ -667,7 +667,7 @@ struct Transcoder { } template - RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool TranscodeUnsafe(InputStream& is, OutputStream& os) { unsigned codepoint; if (!SourceEncoding::Decode(is, &codepoint)) return false; @@ -677,7 +677,7 @@ struct Transcoder { //! Validate one Unicode codepoint from an encoded stream. template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { return Transcode(is, os); // Since source/target encoding is different, must transcode. } }; @@ -690,26 +690,26 @@ inline void PutUnsafe(Stream& stream, typename Stream::Ch c); template struct Transcoder { template - RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Transcode(InputStream& is, OutputStream& os) { os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. return true; } template - RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool TranscodeUnsafe(InputStream& is, OutputStream& os) { PutUnsafe(os, is.Take()); // Just copy one code unit. This semantic is different from primary template class. return true; } template - RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { + static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) { return Encoding::Validate(is, os); // source/target encoding are the same } }; RAPIDJSON_NAMESPACE_END -#if defined(__GNUC__) || defined(_MSC_VER) +#if defined(__GNUC__) || (defined(_MSC_VER) && !defined(__clang__)) RAPIDJSON_DIAG_POP #endif diff --git a/src/3rdparty/rapidjson/error/error.h b/src/3rdparty/rapidjson/error/error.h index 95cb31a7..9311d2f0 100644 --- a/src/3rdparty/rapidjson/error/error.h +++ b/src/3rdparty/rapidjson/error/error.h @@ -104,6 +104,8 @@ enum ParseErrorCode { \see GenericReader::Parse, GenericDocument::Parse */ struct ParseResult { + //!! Unspecified boolean type + typedef bool (ParseResult::*BooleanType)() const; public: //! Default constructor, no error. ParseResult() : code_(kParseErrorNone), offset_(0) {} @@ -115,8 +117,8 @@ public: //! Get the error offset, if \ref IsError(), 0 otherwise. size_t Offset() const { return offset_; } - //! Conversion to \c bool, returns \c true, iff !\ref IsError(). - operator bool() const { return !IsError(); } + //! Explicit conversion to \c bool, returns \c true, iff !\ref IsError(). + operator BooleanType() const { return !IsError() ? &ParseResult::IsError : NULL; } //! Whether the result is an error. bool IsError() const { return code_ != kParseErrorNone; } @@ -124,6 +126,10 @@ public: bool operator==(ParseErrorCode code) const { return code_ == code; } friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; } + bool operator!=(const ParseResult& that) const { return !(*this == that); } + bool operator!=(ParseErrorCode code) const { return !(*this == code); } + friend bool operator!=(ParseErrorCode code, const ParseResult & err) { return err != code; } + //! Reset error code. void Clear() { Set(kParseErrorNone); } //! Update error code and offset. diff --git a/src/3rdparty/rapidjson/filereadstream.h b/src/3rdparty/rapidjson/filereadstream.h index b56ea13b..6b343707 100644 --- a/src/3rdparty/rapidjson/filereadstream.h +++ b/src/3rdparty/rapidjson/filereadstream.h @@ -59,7 +59,7 @@ public: // For encoding detection only. const Ch* Peek4() const { - return (current_ + 4 <= bufferLast_) ? current_ : 0; + return (current_ + 4 - !eof_ <= bufferLast_) ? current_ : 0; } private: @@ -68,7 +68,7 @@ private: ++current_; else if (!eof_) { count_ += readCount_; - readCount_ = fread(buffer_, 1, bufferSize_, fp_); + readCount_ = std::fread(buffer_, 1, bufferSize_, fp_); bufferLast_ = buffer_ + readCount_ - 1; current_ = buffer_; diff --git a/src/3rdparty/rapidjson/filewritestream.h b/src/3rdparty/rapidjson/filewritestream.h index 6378dd60..8b48fee1 100644 --- a/src/3rdparty/rapidjson/filewritestream.h +++ b/src/3rdparty/rapidjson/filewritestream.h @@ -25,7 +25,7 @@ RAPIDJSON_DIAG_OFF(unreachable-code) RAPIDJSON_NAMESPACE_BEGIN -//! Wrapper of C file stream for input using fread(). +//! Wrapper of C file stream for output using fwrite(). /*! \note implements Stream concept */ @@ -62,7 +62,7 @@ public: void Flush() { if (current_ != buffer_) { - size_t result = fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); + size_t result = std::fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); if (result < static_cast(current_ - buffer_)) { // failure deliberately ignored at this time // added to avoid warn_unused_result build errors diff --git a/src/3rdparty/rapidjson/internal/biginteger.h b/src/3rdparty/rapidjson/internal/biginteger.h index 9d3e88c9..a31c8a88 100644 --- a/src/3rdparty/rapidjson/internal/biginteger.h +++ b/src/3rdparty/rapidjson/internal/biginteger.h @@ -17,7 +17,7 @@ #include "../rapidjson.h" -#if defined(_MSC_VER) && defined(_M_AMD64) +#if defined(_MSC_VER) && !__INTEL_COMPILER && defined(_M_AMD64) #include // for _umul128 #pragma intrinsic(_umul128) #endif @@ -133,7 +133,7 @@ public: RAPIDJSON_ASSERT(count_ + offset <= kCapacity); if (interShift == 0) { - std::memmove(&digits_[count_ - 1 + offset], &digits_[count_ - 1], count_ * sizeof(Type)); + std::memmove(digits_ + offset, digits_, count_ * sizeof(Type)); count_ += offset; } else { diff --git a/src/3rdparty/rapidjson/internal/diyfp.h b/src/3rdparty/rapidjson/internal/diyfp.h index c9fefdc6..b6c2cf56 100644 --- a/src/3rdparty/rapidjson/internal/diyfp.h +++ b/src/3rdparty/rapidjson/internal/diyfp.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. // This is a C++ header-only implementation of Grisu2 algorithm from the publication: @@ -20,8 +20,9 @@ #define RAPIDJSON_DIYFP_H_ #include "../rapidjson.h" +#include -#if defined(_MSC_VER) && defined(_M_AMD64) +#if defined(_MSC_VER) && defined(_M_AMD64) && !defined(__INTEL_COMPILER) #include #pragma intrinsic(_BitScanReverse64) #pragma intrinsic(_umul128) @@ -56,7 +57,7 @@ struct DiyFp { if (biased_e != 0) { f = significand + kDpHiddenBit; e = biased_e - kDpExponentBias; - } + } else { f = significand; e = kDpMinExponent + 1; @@ -99,6 +100,7 @@ struct DiyFp { } DiyFp Normalize() const { + RAPIDJSON_ASSERT(f != 0); // https://stackoverflow.com/a/26809183/291737 #if defined(_MSC_VER) && defined(_M_AMD64) unsigned long index; _BitScanReverse64(&index, f); @@ -141,7 +143,16 @@ struct DiyFp { double d; uint64_t u64; }u; - const uint64_t be = (e == kDpDenormalExponent && (f & kDpHiddenBit) == 0) ? 0 : + RAPIDJSON_ASSERT(f <= kDpHiddenBit + kDpSignificandMask); + if (e < kDpDenormalExponent) { + // Underflow. + return 0.0; + } + if (e >= kDpMaxExponent) { + // Overflow. + return std::numeric_limits::infinity(); + } + const uint64_t be = (e == kDpDenormalExponent && (f & kDpHiddenBit) == 0) ? 0 : static_cast(e + kDpExponentBias); u.u64 = (f & kDpSignificandMask) | (be << kDpSignificandSize); return u.d; @@ -220,9 +231,10 @@ inline DiyFp GetCachedPowerByIndex(size_t index) { 641, 667, 694, 720, 747, 774, 800, 827, 853, 880, 907, 933, 960, 986, 1013, 1039, 1066 }; + RAPIDJSON_ASSERT(index < 87); return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]); } - + inline DiyFp GetCachedPower(int e, int* K) { //int k = static_cast(ceil((-61 - e) * 0.30102999566398114)) + 374; @@ -238,10 +250,11 @@ inline DiyFp GetCachedPower(int e, int* K) { } inline DiyFp GetCachedPower10(int exp, int *outExp) { - unsigned index = (static_cast(exp) + 348u) / 8u; - *outExp = -348 + static_cast(index) * 8; - return GetCachedPowerByIndex(index); - } + RAPIDJSON_ASSERT(exp >= -348); + unsigned index = static_cast(exp + 348) / 8u; + *outExp = -348 + static_cast(index) * 8; + return GetCachedPowerByIndex(index); +} #ifdef __GNUC__ RAPIDJSON_DIAG_POP diff --git a/src/3rdparty/rapidjson/internal/dtoa.h b/src/3rdparty/rapidjson/internal/dtoa.h index 8d6350e6..bf2e9b2e 100644 --- a/src/3rdparty/rapidjson/internal/dtoa.h +++ b/src/3rdparty/rapidjson/internal/dtoa.h @@ -41,7 +41,7 @@ inline void GrisuRound(char* buffer, int len, uint64_t delta, uint64_t rest, uin } } -inline unsigned CountDecimalDigit32(uint32_t n) { +inline int CountDecimalDigit32(uint32_t n) { // Simple pure C++ implementation was faster than __builtin_clz version in this situation. if (n < 10) return 1; if (n < 100) return 2; @@ -63,7 +63,7 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff const DiyFp wp_w = Mp - W; uint32_t p1 = static_cast(Mp.f >> -one.e); uint64_t p2 = Mp.f & (one.f - 1); - unsigned kappa = CountDecimalDigit32(p1); // kappa in [0, 9] + int kappa = CountDecimalDigit32(p1); // kappa in [0, 9] *len = 0; while (kappa > 0) { @@ -102,8 +102,8 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff kappa--; if (p2 < delta) { *K += kappa; - int index = -static_cast(kappa); - GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[-static_cast(kappa)] : 0)); + int index = -kappa; + GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[index] : 0)); return; } } diff --git a/src/3rdparty/rapidjson/internal/ieee754.h b/src/3rdparty/rapidjson/internal/ieee754.h index 82bb0b99..c2684ba2 100644 --- a/src/3rdparty/rapidjson/internal/ieee754.h +++ b/src/3rdparty/rapidjson/internal/ieee754.h @@ -48,13 +48,13 @@ public: int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } - static unsigned EffectiveSignificandSize(int order) { + static int EffectiveSignificandSize(int order) { if (order >= -1021) return 53; else if (order <= -1074) return 0; else - return static_cast(order) + 1074; + return order + 1074; } private: diff --git a/src/3rdparty/rapidjson/internal/itoa.h b/src/3rdparty/rapidjson/internal/itoa.h index 01a4e7e7..9b1c45cc 100644 --- a/src/3rdparty/rapidjson/internal/itoa.h +++ b/src/3rdparty/rapidjson/internal/itoa.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ITOA_ @@ -37,12 +37,14 @@ inline const char* GetDigitsLut() { } inline char* u32toa(uint32_t value, char* buffer) { + RAPIDJSON_ASSERT(buffer != 0); + const char* cDigitsLut = GetDigitsLut(); if (value < 10000) { const uint32_t d1 = (value / 100) << 1; const uint32_t d2 = (value % 100) << 1; - + if (value >= 1000) *buffer++ = cDigitsLut[d1]; if (value >= 100) @@ -55,13 +57,13 @@ inline char* u32toa(uint32_t value, char* buffer) { // value = bbbbcccc const uint32_t b = value / 10000; const uint32_t c = value % 10000; - + const uint32_t d1 = (b / 100) << 1; const uint32_t d2 = (b % 100) << 1; - + const uint32_t d3 = (c / 100) << 1; const uint32_t d4 = (c % 100) << 1; - + if (value >= 10000000) *buffer++ = cDigitsLut[d1]; if (value >= 1000000) @@ -69,7 +71,7 @@ inline char* u32toa(uint32_t value, char* buffer) { if (value >= 100000) *buffer++ = cDigitsLut[d2]; *buffer++ = cDigitsLut[d2 + 1]; - + *buffer++ = cDigitsLut[d3]; *buffer++ = cDigitsLut[d3 + 1]; *buffer++ = cDigitsLut[d4]; @@ -77,10 +79,10 @@ inline char* u32toa(uint32_t value, char* buffer) { } else { // value = aabbbbcccc in decimal - + const uint32_t a = value / 100000000; // 1 to 42 value %= 100000000; - + if (a >= 10) { const unsigned i = a << 1; *buffer++ = cDigitsLut[i]; @@ -91,13 +93,13 @@ inline char* u32toa(uint32_t value, char* buffer) { const uint32_t b = value / 10000; // 0 to 9999 const uint32_t c = value % 10000; // 0 to 9999 - + const uint32_t d1 = (b / 100) << 1; const uint32_t d2 = (b % 100) << 1; - + const uint32_t d3 = (c / 100) << 1; const uint32_t d4 = (c % 100) << 1; - + *buffer++ = cDigitsLut[d1]; *buffer++ = cDigitsLut[d1 + 1]; *buffer++ = cDigitsLut[d2]; @@ -111,6 +113,7 @@ inline char* u32toa(uint32_t value, char* buffer) { } inline char* i32toa(int32_t value, char* buffer) { + RAPIDJSON_ASSERT(buffer != 0); uint32_t u = static_cast(value); if (value < 0) { *buffer++ = '-'; @@ -121,6 +124,7 @@ inline char* i32toa(int32_t value, char* buffer) { } inline char* u64toa(uint64_t value, char* buffer) { + RAPIDJSON_ASSERT(buffer != 0); const char* cDigitsLut = GetDigitsLut(); const uint64_t kTen8 = 100000000; const uint64_t kTen9 = kTen8 * 10; @@ -131,13 +135,13 @@ inline char* u64toa(uint64_t value, char* buffer) { const uint64_t kTen14 = kTen8 * 1000000; const uint64_t kTen15 = kTen8 * 10000000; const uint64_t kTen16 = kTen8 * kTen8; - + if (value < kTen8) { uint32_t v = static_cast(value); if (v < 10000) { const uint32_t d1 = (v / 100) << 1; const uint32_t d2 = (v % 100) << 1; - + if (v >= 1000) *buffer++ = cDigitsLut[d1]; if (v >= 100) @@ -150,13 +154,13 @@ inline char* u64toa(uint64_t value, char* buffer) { // value = bbbbcccc const uint32_t b = v / 10000; const uint32_t c = v % 10000; - + const uint32_t d1 = (b / 100) << 1; const uint32_t d2 = (b % 100) << 1; - + const uint32_t d3 = (c / 100) << 1; const uint32_t d4 = (c % 100) << 1; - + if (value >= 10000000) *buffer++ = cDigitsLut[d1]; if (value >= 1000000) @@ -164,7 +168,7 @@ inline char* u64toa(uint64_t value, char* buffer) { if (value >= 100000) *buffer++ = cDigitsLut[d2]; *buffer++ = cDigitsLut[d2 + 1]; - + *buffer++ = cDigitsLut[d3]; *buffer++ = cDigitsLut[d3 + 1]; *buffer++ = cDigitsLut[d4]; @@ -174,22 +178,22 @@ inline char* u64toa(uint64_t value, char* buffer) { else if (value < kTen16) { const uint32_t v0 = static_cast(value / kTen8); const uint32_t v1 = static_cast(value % kTen8); - + const uint32_t b0 = v0 / 10000; const uint32_t c0 = v0 % 10000; - + const uint32_t d1 = (b0 / 100) << 1; const uint32_t d2 = (b0 % 100) << 1; - + const uint32_t d3 = (c0 / 100) << 1; const uint32_t d4 = (c0 % 100) << 1; const uint32_t b1 = v1 / 10000; const uint32_t c1 = v1 % 10000; - + const uint32_t d5 = (b1 / 100) << 1; const uint32_t d6 = (b1 % 100) << 1; - + const uint32_t d7 = (c1 / 100) << 1; const uint32_t d8 = (c1 % 100) << 1; @@ -207,9 +211,8 @@ inline char* u64toa(uint64_t value, char* buffer) { *buffer++ = cDigitsLut[d3 + 1]; if (value >= kTen9) *buffer++ = cDigitsLut[d4]; - if (value >= kTen8) - *buffer++ = cDigitsLut[d4 + 1]; - + + *buffer++ = cDigitsLut[d4 + 1]; *buffer++ = cDigitsLut[d5]; *buffer++ = cDigitsLut[d5 + 1]; *buffer++ = cDigitsLut[d6]; @@ -222,7 +225,7 @@ inline char* u64toa(uint64_t value, char* buffer) { else { const uint32_t a = static_cast(value / kTen16); // 1 to 1844 value %= kTen16; - + if (a < 10) *buffer++ = static_cast('0' + static_cast(a)); else if (a < 100) { @@ -232,7 +235,7 @@ inline char* u64toa(uint64_t value, char* buffer) { } else if (a < 1000) { *buffer++ = static_cast('0' + static_cast(a / 100)); - + const uint32_t i = (a % 100) << 1; *buffer++ = cDigitsLut[i]; *buffer++ = cDigitsLut[i + 1]; @@ -245,28 +248,28 @@ inline char* u64toa(uint64_t value, char* buffer) { *buffer++ = cDigitsLut[j]; *buffer++ = cDigitsLut[j + 1]; } - + const uint32_t v0 = static_cast(value / kTen8); const uint32_t v1 = static_cast(value % kTen8); - + const uint32_t b0 = v0 / 10000; const uint32_t c0 = v0 % 10000; - + const uint32_t d1 = (b0 / 100) << 1; const uint32_t d2 = (b0 % 100) << 1; - + const uint32_t d3 = (c0 / 100) << 1; const uint32_t d4 = (c0 % 100) << 1; - + const uint32_t b1 = v1 / 10000; const uint32_t c1 = v1 % 10000; - + const uint32_t d5 = (b1 / 100) << 1; const uint32_t d6 = (b1 % 100) << 1; - + const uint32_t d7 = (c1 / 100) << 1; const uint32_t d8 = (c1 % 100) << 1; - + *buffer++ = cDigitsLut[d1]; *buffer++ = cDigitsLut[d1 + 1]; *buffer++ = cDigitsLut[d2]; @@ -284,11 +287,12 @@ inline char* u64toa(uint64_t value, char* buffer) { *buffer++ = cDigitsLut[d8]; *buffer++ = cDigitsLut[d8 + 1]; } - + return buffer; } inline char* i64toa(int64_t value, char* buffer) { + RAPIDJSON_ASSERT(buffer != 0); uint64_t u = static_cast(value); if (value < 0) { *buffer++ = '-'; diff --git a/src/3rdparty/rapidjson/internal/meta.h b/src/3rdparty/rapidjson/internal/meta.h index 5a9aaa42..d401edf8 100644 --- a/src/3rdparty/rapidjson/internal/meta.h +++ b/src/3rdparty/rapidjson/internal/meta.h @@ -21,7 +21,8 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif -#if defined(_MSC_VER) + +#if defined(_MSC_VER) && !defined(__clang__) RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(6334) #endif @@ -174,7 +175,11 @@ template struct RemoveSfinaeTag { typedef T Type; RAPIDJSON_NAMESPACE_END //@endcond -#if defined(__GNUC__) || defined(_MSC_VER) +#if defined(_MSC_VER) && !defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + +#ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif diff --git a/src/3rdparty/rapidjson/internal/regex.h b/src/3rdparty/rapidjson/internal/regex.h index 422a5240..16e35592 100644 --- a/src/3rdparty/rapidjson/internal/regex.h +++ b/src/3rdparty/rapidjson/internal/regex.h @@ -24,16 +24,17 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) RAPIDJSON_DIAG_OFF(switch-enum) RAPIDJSON_DIAG_OFF(implicit-fallthrough) +#elif defined(_MSC_VER) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated #endif #ifdef __GNUC__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) +#if __GNUC__ >= 7 +RAPIDJSON_DIAG_OFF(implicit-fallthrough) #endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated #endif #ifndef RAPIDJSON_REGEX_VERBOSE @@ -43,12 +44,40 @@ RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated RAPIDJSON_NAMESPACE_BEGIN namespace internal { +/////////////////////////////////////////////////////////////////////////////// +// DecodedStream + +template +class DecodedStream { +public: + DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } + unsigned Peek() { return codepoint_; } + unsigned Take() { + unsigned c = codepoint_; + if (c) // No further decoding when '\0' + Decode(); + return c; + } + +private: + void Decode() { + if (!Encoding::Decode(ss_, &codepoint_)) + codepoint_ = 0; + } + + SourceStream& ss_; + unsigned codepoint_; +}; + /////////////////////////////////////////////////////////////////////////////// // GenericRegex static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 static const SizeType kRegexInvalidRange = ~SizeType(0); +template +class GenericRegexSearch; + //! Regular expression engine with subset of ECMAscript grammar. /*! Supported regular expression syntax: @@ -84,45 +113,29 @@ static const SizeType kRegexInvalidRange = ~SizeType(0); template class GenericRegex { public: + typedef Encoding EncodingType; typedef typename Encoding::Ch Ch; + template friend class GenericRegexSearch; GenericRegex(const Ch* source, Allocator* allocator = 0) : - states_(allocator, 256), ranges_(allocator, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), - stateSet_(), state0_(allocator, 0), state1_(allocator, 0), anchorBegin_(), anchorEnd_() + ownAllocator_(allocator ? 0 : RAPIDJSON_NEW(Allocator)()), allocator_(allocator ? allocator : ownAllocator_), + states_(allocator_, 256), ranges_(allocator_, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), + anchorBegin_(), anchorEnd_() { GenericStringStream ss(source); - DecodedStream > ds(ss); + DecodedStream, Encoding> ds(ss); Parse(ds); } - ~GenericRegex() { - Allocator::Free(stateSet_); + ~GenericRegex() + { + RAPIDJSON_DELETE(ownAllocator_); } bool IsValid() const { return root_ != kRegexInvalidState; } - template - bool Match(InputStream& is) const { - return SearchWithAnchoring(is, true, true); - } - - bool Match(const Ch* s) const { - GenericStringStream is(s); - return Match(is); - } - - template - bool Search(InputStream& is) const { - return SearchWithAnchoring(is, anchorBegin_, anchorEnd_); - } - - bool Search(const Ch* s) const { - GenericStringStream is(s); - return Search(is); - } - private: enum Operator { kZeroOrOne, @@ -157,28 +170,6 @@ private: SizeType minIndex; }; - template - class DecodedStream { - public: - DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } - unsigned Peek() { return codepoint_; } - unsigned Take() { - unsigned c = codepoint_; - if (c) // No further decoding when '\0' - Decode(); - return c; - } - - private: - void Decode() { - if (!Encoding::Decode(ss_, &codepoint_)) - codepoint_ = 0; - } - - SourceStream& ss_; - unsigned codepoint_; - }; - State& GetState(SizeType index) { RAPIDJSON_ASSERT(index < stateCount_); return states_.template Bottom()[index]; @@ -200,11 +191,10 @@ private: } template - void Parse(DecodedStream& ds) { - Allocator allocator; - Stack operandStack(&allocator, 256); // Frag - Stack operatorStack(&allocator, 256); // Operator - Stack atomCountStack(&allocator, 256); // unsigned (Atom per parenthesis) + void Parse(DecodedStream& ds) { + Stack operandStack(allocator_, 256); // Frag + Stack operatorStack(allocator_, 256); // Operator + Stack atomCountStack(allocator_, 256); // unsigned (Atom per parenthesis) *atomCountStack.template Push() = 0; @@ -327,14 +317,6 @@ private: printf("\n"); #endif } - - // Preallocate buffer for SearchWithAnchoring() - RAPIDJSON_ASSERT(stateSet_ == 0); - if (stateCount_ > 0) { - stateSet_ = static_cast(states_.GetAllocator().Malloc(GetStateSetSize())); - state0_.template Reserve(stateCount_); - state1_.template Reserve(stateCount_); - } } SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) { @@ -413,8 +395,7 @@ private: } return false; - default: - RAPIDJSON_ASSERT(op == kOneOrMore); + case kOneOrMore: if (operandStack.GetSize() >= sizeof(Frag)) { Frag e = *operandStack.template Pop(1); SizeType s = NewState(kRegexInvalidState, e.start, 0); @@ -423,6 +404,10 @@ private: return true; } return false; + + default: + // syntax error (e.g. unclosed kLeftParenthesis) + return false; } } @@ -483,7 +468,7 @@ private: } template - bool ParseUnsigned(DecodedStream& ds, unsigned* u) { + bool ParseUnsigned(DecodedStream& ds, unsigned* u) { unsigned r = 0; if (ds.Peek() < '0' || ds.Peek() > '9') return false; @@ -497,7 +482,7 @@ private: } template - bool ParseRange(DecodedStream& ds, SizeType* range) { + bool ParseRange(DecodedStream& ds, SizeType* range) { bool isBegin = true; bool negate = false; int step = 0; @@ -575,7 +560,7 @@ private: } template - bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) { + bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) { unsigned codepoint; switch (codepoint = ds.Take()) { case '^': @@ -603,72 +588,8 @@ private: } } - template - bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) const { - RAPIDJSON_ASSERT(IsValid()); - DecodedStream ds(is); - - state0_.Clear(); - Stack *current = &state0_, *next = &state1_; - const size_t stateSetSize = GetStateSetSize(); - std::memset(stateSet_, 0, stateSetSize); - - bool matched = AddState(*current, root_); - unsigned codepoint; - while (!current->Empty() && (codepoint = ds.Take()) != 0) { - std::memset(stateSet_, 0, stateSetSize); - next->Clear(); - matched = false; - for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) { - const State& sr = GetState(*s); - if (sr.codepoint == codepoint || - sr.codepoint == kAnyCharacterClass || - (sr.codepoint == kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) - { - matched = AddState(*next, sr.out) || matched; - if (!anchorEnd && matched) - return true; - } - if (!anchorBegin) - AddState(*next, root_); - } - internal::Swap(current, next); - } - - return matched; - } - - size_t GetStateSetSize() const { - return (stateCount_ + 31) / 32 * 4; - } - - // Return whether the added states is a match state - bool AddState(Stack& l, SizeType index) const { - RAPIDJSON_ASSERT(index != kRegexInvalidState); - - const State& s = GetState(index); - if (s.out1 != kRegexInvalidState) { // Split - bool matched = AddState(l, s.out); - return AddState(l, s.out1) || matched; - } - else if (!(stateSet_[index >> 5] & (1 << (index & 31)))) { - stateSet_[index >> 5] |= (1 << (index & 31)); - *l.template PushUnsafe() = index; - } - return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. - } - - bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { - bool yes = (GetRange(rangeIndex).start & kRangeNegationFlag) == 0; - while (rangeIndex != kRegexInvalidRange) { - const Range& r = GetRange(rangeIndex); - if (codepoint >= (r.start & ~kRangeNegationFlag) && codepoint <= r.end) - return yes; - rangeIndex = r.next; - } - return !yes; - } - + Allocator* ownAllocator_; + Allocator* allocator_; Stack states_; Stack ranges_; SizeType root_; @@ -678,23 +599,141 @@ private: static const unsigned kInfinityQuantifier = ~0u; // For SearchWithAnchoring() - uint32_t* stateSet_; // allocated by states_.GetAllocator() - mutable Stack state0_; - mutable Stack state1_; bool anchorBegin_; bool anchorEnd_; }; +template +class GenericRegexSearch { +public: + typedef typename RegexType::EncodingType Encoding; + typedef typename Encoding::Ch Ch; + + GenericRegexSearch(const RegexType& regex, Allocator* allocator = 0) : + regex_(regex), allocator_(allocator), ownAllocator_(0), + state0_(allocator, 0), state1_(allocator, 0), stateSet_() + { + RAPIDJSON_ASSERT(regex_.IsValid()); + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + stateSet_ = static_cast(allocator_->Malloc(GetStateSetSize())); + state0_.template Reserve(regex_.stateCount_); + state1_.template Reserve(regex_.stateCount_); + } + + ~GenericRegexSearch() { + Allocator::Free(stateSet_); + RAPIDJSON_DELETE(ownAllocator_); + } + + template + bool Match(InputStream& is) { + return SearchWithAnchoring(is, true, true); + } + + bool Match(const Ch* s) { + GenericStringStream is(s); + return Match(is); + } + + template + bool Search(InputStream& is) { + return SearchWithAnchoring(is, regex_.anchorBegin_, regex_.anchorEnd_); + } + + bool Search(const Ch* s) { + GenericStringStream is(s); + return Search(is); + } + +private: + typedef typename RegexType::State State; + typedef typename RegexType::Range Range; + + template + bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) { + DecodedStream ds(is); + + state0_.Clear(); + Stack *current = &state0_, *next = &state1_; + const size_t stateSetSize = GetStateSetSize(); + std::memset(stateSet_, 0, stateSetSize); + + bool matched = AddState(*current, regex_.root_); + unsigned codepoint; + while (!current->Empty() && (codepoint = ds.Take()) != 0) { + std::memset(stateSet_, 0, stateSetSize); + next->Clear(); + matched = false; + for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) { + const State& sr = regex_.GetState(*s); + if (sr.codepoint == codepoint || + sr.codepoint == RegexType::kAnyCharacterClass || + (sr.codepoint == RegexType::kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) + { + matched = AddState(*next, sr.out) || matched; + if (!anchorEnd && matched) + return true; + } + if (!anchorBegin) + AddState(*next, regex_.root_); + } + internal::Swap(current, next); + } + + return matched; + } + + size_t GetStateSetSize() const { + return (regex_.stateCount_ + 31) / 32 * 4; + } + + // Return whether the added states is a match state + bool AddState(Stack& l, SizeType index) { + RAPIDJSON_ASSERT(index != kRegexInvalidState); + + const State& s = regex_.GetState(index); + if (s.out1 != kRegexInvalidState) { // Split + bool matched = AddState(l, s.out); + return AddState(l, s.out1) || matched; + } + else if (!(stateSet_[index >> 5] & (1u << (index & 31)))) { + stateSet_[index >> 5] |= (1u << (index & 31)); + *l.template PushUnsafe() = index; + } + return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. + } + + bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { + bool yes = (regex_.GetRange(rangeIndex).start & RegexType::kRangeNegationFlag) == 0; + while (rangeIndex != kRegexInvalidRange) { + const Range& r = regex_.GetRange(rangeIndex); + if (codepoint >= (r.start & ~RegexType::kRangeNegationFlag) && codepoint <= r.end) + return yes; + rangeIndex = r.next; + } + return !yes; + } + + const RegexType& regex_; + Allocator* allocator_; + Allocator* ownAllocator_; + Stack state0_; + Stack state1_; + uint32_t* stateSet_; +}; + typedef GenericRegex > Regex; +typedef GenericRegexSearch RegexSearch; } // namespace internal RAPIDJSON_NAMESPACE_END -#ifdef __clang__ +#ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif -#ifdef _MSC_VER +#if defined(__clang__) || defined(_MSC_VER) RAPIDJSON_DIAG_POP #endif diff --git a/src/3rdparty/rapidjson/internal/stack.h b/src/3rdparty/rapidjson/internal/stack.h index 022c9aab..45dca6a8 100644 --- a/src/3rdparty/rapidjson/internal/stack.h +++ b/src/3rdparty/rapidjson/internal/stack.h @@ -17,6 +17,7 @@ #include "../allocators.h" #include "swap.h" +#include #if defined(__clang__) RAPIDJSON_DIAG_PUSH @@ -100,7 +101,7 @@ public: void ShrinkToFit() { if (Empty()) { // If the stack is empty, completely deallocate the memory. - Allocator::Free(stack_); + Allocator::Free(stack_); // NOLINT (+clang-analyzer-unix.Malloc) stack_ = 0; stackTop_ = 0; stackEnd_ = 0; @@ -114,7 +115,7 @@ public: template RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { // Expand the stack if needed - if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) + if (RAPIDJSON_UNLIKELY(static_cast(sizeof(T) * count) > (stackEnd_ - stackTop_))) Expand(count); } @@ -126,7 +127,8 @@ public: template RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { - RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); + RAPIDJSON_ASSERT(stackTop_); + RAPIDJSON_ASSERT(static_cast(sizeof(T) * count) <= (stackEnd_ - stackTop_)); T* ret = reinterpret_cast(stackTop_); stackTop_ += sizeof(T) * count; return ret; @@ -183,7 +185,7 @@ private: size_t newCapacity; if (stack_ == 0) { if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); newCapacity = initialCapacity_; } else { newCapacity = GetCapacity(); diff --git a/src/3rdparty/rapidjson/internal/strfunc.h b/src/3rdparty/rapidjson/internal/strfunc.h index 2edfae52..226439a7 100644 --- a/src/3rdparty/rapidjson/internal/strfunc.h +++ b/src/3rdparty/rapidjson/internal/strfunc.h @@ -16,6 +16,7 @@ #define RAPIDJSON_INTERNAL_STRFUNC_H_ #include "../stream.h" +#include RAPIDJSON_NAMESPACE_BEGIN namespace internal { @@ -28,14 +29,27 @@ namespace internal { */ template inline SizeType StrLen(const Ch* s) { + RAPIDJSON_ASSERT(s != 0); const Ch* p = s; while (*p) ++p; return SizeType(p - s); } +template <> +inline SizeType StrLen(const char* s) { + return SizeType(std::strlen(s)); +} + +template <> +inline SizeType StrLen(const wchar_t* s) { + return SizeType(std::wcslen(s)); +} + //! Returns number of code points in a encoded string. template bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { + RAPIDJSON_ASSERT(s != 0); + RAPIDJSON_ASSERT(outCount != 0); GenericStringStream is(s); const typename Encoding::Ch* end = s + length; SizeType count = 0; diff --git a/src/3rdparty/rapidjson/internal/strtod.h b/src/3rdparty/rapidjson/internal/strtod.h index 289c413b..dfca22b6 100644 --- a/src/3rdparty/rapidjson/internal/strtod.h +++ b/src/3rdparty/rapidjson/internal/strtod.h @@ -19,6 +19,8 @@ #include "biginteger.h" #include "diyfp.h" #include "pow10.h" +#include +#include RAPIDJSON_NAMESPACE_BEGIN namespace internal { @@ -126,46 +128,46 @@ inline bool StrtodFast(double d, int p, double* result) { } // Compute an approximation and see if it is within 1/2 ULP -inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosition, int exp, double* result) { +inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result) { uint64_t significand = 0; - size_t i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 - for (; i < length; i++) { + int i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 + for (; i < dLen; i++) { if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) break; significand = significand * 10u + static_cast(decimals[i] - '0'); } - if (i < length && decimals[i] >= '5') // Rounding + if (i < dLen && decimals[i] >= '5') // Rounding significand++; - size_t remaining = length - i; - const unsigned kUlpShift = 3; - const unsigned kUlp = 1 << kUlpShift; + int remaining = dLen - i; + const int kUlpShift = 3; + const int kUlp = 1 << kUlpShift; int64_t error = (remaining == 0) ? 0 : kUlp / 2; DiyFp v(significand, 0); v = v.Normalize(); error <<= -v.e; - const int dExp = static_cast(decimalPosition) - static_cast(i) + exp; + dExp += remaining; int actualExp; DiyFp cachedPower = GetCachedPower10(dExp, &actualExp); if (actualExp != dExp) { static const DiyFp kPow10[] = { - DiyFp(RAPIDJSON_UINT64_C2(0xa0000000, 00000000), -60), // 10^1 - DiyFp(RAPIDJSON_UINT64_C2(0xc8000000, 00000000), -57), // 10^2 - DiyFp(RAPIDJSON_UINT64_C2(0xfa000000, 00000000), -54), // 10^3 - DiyFp(RAPIDJSON_UINT64_C2(0x9c400000, 00000000), -50), // 10^4 - DiyFp(RAPIDJSON_UINT64_C2(0xc3500000, 00000000), -47), // 10^5 - DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 00000000), -44), // 10^6 - DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 00000000), -40) // 10^7 + DiyFp(RAPIDJSON_UINT64_C2(0xa0000000, 0x00000000), -60), // 10^1 + DiyFp(RAPIDJSON_UINT64_C2(0xc8000000, 0x00000000), -57), // 10^2 + DiyFp(RAPIDJSON_UINT64_C2(0xfa000000, 0x00000000), -54), // 10^3 + DiyFp(RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), -50), // 10^4 + DiyFp(RAPIDJSON_UINT64_C2(0xc3500000, 0x00000000), -47), // 10^5 + DiyFp(RAPIDJSON_UINT64_C2(0xf4240000, 0x00000000), -44), // 10^6 + DiyFp(RAPIDJSON_UINT64_C2(0x98968000, 0x00000000), -40) // 10^7 }; - int adjustment = dExp - actualExp - 1; - RAPIDJSON_ASSERT(adjustment >= 0 && adjustment < 7); - v = v * kPow10[adjustment]; - if (length + static_cast(adjustment)> 19u) // has more digits than decimal digits in 64-bit + int adjustment = dExp - actualExp; + RAPIDJSON_ASSERT(adjustment >= 1 && adjustment < 8); + v = v * kPow10[adjustment - 1]; + if (dLen + adjustment > 19) // has more digits than decimal digits in 64-bit error += kUlp / 2; } @@ -177,17 +179,17 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit v = v.Normalize(); error <<= oldExp - v.e; - const unsigned effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); - unsigned precisionSize = 64 - effectiveSignificandSize; + const int effectiveSignificandSize = Double::EffectiveSignificandSize(64 + v.e); + int precisionSize = 64 - effectiveSignificandSize; if (precisionSize + kUlpShift >= 64) { - unsigned scaleExp = (precisionSize + kUlpShift) - 63; + int scaleExp = (precisionSize + kUlpShift) - 63; v.f >>= scaleExp; v.e += scaleExp; - error = (error >> scaleExp) + 1 + static_cast(kUlp); + error = (error >> scaleExp) + 1 + kUlp; precisionSize -= scaleExp; } - DiyFp rounded(v.f >> precisionSize, v.e + static_cast(precisionSize)); + DiyFp rounded(v.f >> precisionSize, v.e + precisionSize); const uint64_t precisionBits = (v.f & ((uint64_t(1) << precisionSize) - 1)) * kUlp; const uint64_t halfWay = (uint64_t(1) << (precisionSize - 1)) * kUlp; if (precisionBits >= halfWay + static_cast(error)) { @@ -203,9 +205,9 @@ inline bool StrtodDiyFp(const char* decimals, size_t length, size_t decimalPosit return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); } -inline double StrtodBigInteger(double approx, const char* decimals, size_t length, size_t decimalPosition, int exp) { - const BigInteger dInt(decimals, length); - const int dExp = static_cast(decimalPosition) - static_cast(length) + exp; +inline double StrtodBigInteger(double approx, const char* decimals, int dLen, int dExp) { + RAPIDJSON_ASSERT(dLen >= 0); + const BigInteger dInt(decimals, static_cast(dLen)); Double a(approx); int cmp = CheckWithinHalfULP(a.Value(), dInt, dExp); if (cmp < 0) @@ -225,42 +227,61 @@ inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t RAPIDJSON_ASSERT(d >= 0.0); RAPIDJSON_ASSERT(length >= 1); - double result; + double result = 0.0; if (StrtodFast(d, p, &result)) return result; + RAPIDJSON_ASSERT(length <= INT_MAX); + int dLen = static_cast(length); + + RAPIDJSON_ASSERT(length >= decimalPosition); + RAPIDJSON_ASSERT(length - decimalPosition <= INT_MAX); + int dExpAdjust = static_cast(length - decimalPosition); + + RAPIDJSON_ASSERT(exp >= INT_MIN + dExpAdjust); + int dExp = exp - dExpAdjust; + + // Make sure length+dExp does not overflow + RAPIDJSON_ASSERT(dExp <= INT_MAX - dLen); + // Trim leading zeros - while (*decimals == '0' && length > 1) { - length--; + while (dLen > 0 && *decimals == '0') { + dLen--; decimals++; - decimalPosition--; } // Trim trailing zeros - while (decimals[length - 1] == '0' && length > 1) { - length--; - decimalPosition--; - exp++; + while (dLen > 0 && decimals[dLen - 1] == '0') { + dLen--; + dExp++; + } + + if (dLen == 0) { // Buffer only contains zeros. + return 0.0; } // Trim right-most digits - const int kMaxDecimalDigit = 780; - if (static_cast(length) > kMaxDecimalDigit) { - int delta = (static_cast(length) - kMaxDecimalDigit); - exp += delta; - decimalPosition -= static_cast(delta); - length = kMaxDecimalDigit; + const int kMaxDecimalDigit = 767 + 1; + if (dLen > kMaxDecimalDigit) { + dExp += dLen - kMaxDecimalDigit; + dLen = kMaxDecimalDigit; } - // If too small, underflow to zero - if (int(length) + exp < -324) + // If too small, underflow to zero. + // Any x <= 10^-324 is interpreted as zero. + if (dLen + dExp <= -324) return 0.0; - if (StrtodDiyFp(decimals, length, decimalPosition, exp, &result)) + // If too large, overflow to infinity. + // Any x >= 10^309 is interpreted as +infinity. + if (dLen + dExp > 309) + return std::numeric_limits::infinity(); + + if (StrtodDiyFp(decimals, dLen, dExp, &result)) return result; // Use approximation from StrtodDiyFp and make adjustment with BigInteger comparison - return StrtodBigInteger(result, decimals, length, decimalPosition, exp); + return StrtodBigInteger(result, decimals, dLen, dExp); } } // namespace internal diff --git a/src/3rdparty/rapidjson/istreamwrapper.h b/src/3rdparty/rapidjson/istreamwrapper.h index f5fe2897..c4950b9d 100644 --- a/src/3rdparty/rapidjson/istreamwrapper.h +++ b/src/3rdparty/rapidjson/istreamwrapper.h @@ -17,13 +17,12 @@ #include "stream.h" #include +#include #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) -#endif - -#ifdef _MSC_VER +#elif defined(_MSC_VER) RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4351) // new behavior: elements of array 'array' will be default initialized #endif @@ -50,57 +49,71 @@ template class BasicIStreamWrapper { public: typedef typename StreamType::char_type Ch; - BasicIStreamWrapper(StreamType& stream) : stream_(stream), count_(), peekBuffer_() {} - Ch Peek() const { - typename StreamType::int_type c = stream_.peek(); - return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast(c) : '\0'; + //! Constructor. + /*! + \param stream stream opened for read. + */ + BasicIStreamWrapper(StreamType &stream) : stream_(stream), buffer_(peekBuffer_), bufferSize_(4), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { + Read(); } - Ch Take() { - typename StreamType::int_type c = stream_.get(); - if (RAPIDJSON_LIKELY(c != StreamType::traits_type::eof())) { - count_++; - return static_cast(c); - } - else - return '\0'; + //! Constructor. + /*! + \param stream stream opened for read. + \param buffer user-supplied buffer. + \param bufferSize size of buffer in bytes. Must >=4 bytes. + */ + BasicIStreamWrapper(StreamType &stream, char* buffer, size_t bufferSize) : stream_(stream), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { + RAPIDJSON_ASSERT(bufferSize >= 4); + Read(); } - // tellg() may return -1 when failed. So we count by ourself. - size_t Tell() const { return count_; } + Ch Peek() const { return *current_; } + Ch Take() { Ch c = *current_; Read(); return c; } + size_t Tell() const { return count_ + static_cast(current_ - buffer_); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + // Not implemented void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } // For encoding detection only. const Ch* Peek4() const { - RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream. - int i; - bool hasError = false; - for (i = 0; i < 4; ++i) { - typename StreamType::int_type c = stream_.get(); - if (c == StreamType::traits_type::eof()) { - hasError = true; - stream_.clear(); - break; - } - peekBuffer_[i] = static_cast(c); - } - for (--i; i >= 0; --i) - stream_.putback(peekBuffer_[i]); - return !hasError ? peekBuffer_ : 0; + return (current_ + 4 - !eof_ <= bufferLast_) ? current_ : 0; } private: + BasicIStreamWrapper(); BasicIStreamWrapper(const BasicIStreamWrapper&); BasicIStreamWrapper& operator=(const BasicIStreamWrapper&); - StreamType& stream_; - size_t count_; //!< Number of characters read. Note: - mutable Ch peekBuffer_[4]; + void Read() { + if (current_ < bufferLast_) + ++current_; + else if (!eof_) { + count_ += readCount_; + readCount_ = bufferSize_; + bufferLast_ = buffer_ + readCount_ - 1; + current_ = buffer_; + + if (!stream_.read(buffer_, static_cast(bufferSize_))) { + readCount_ = static_cast(stream_.gcount()); + *(bufferLast_ = buffer_ + readCount_) = '\0'; + eof_ = true; + } + } + } + + StreamType &stream_; + Ch peekBuffer_[4], *buffer_; + size_t bufferSize_; + Ch *bufferLast_; + Ch *current_; + size_t readCount_; + size_t count_; //!< Number of characters read + bool eof_; }; typedef BasicIStreamWrapper IStreamWrapper; diff --git a/src/3rdparty/rapidjson/license.txt b/src/3rdparty/rapidjson/license.txt new file mode 100644 index 00000000..7ccc161c --- /dev/null +++ b/src/3rdparty/rapidjson/license.txt @@ -0,0 +1,57 @@ +Tencent is pleased to support the open source community by making RapidJSON available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. + +If you have downloaded a copy of the RapidJSON binary from Tencent, please note that the RapidJSON binary is licensed under the MIT License. +If you have downloaded a copy of the RapidJSON source code from Tencent, please note that RapidJSON source code is licensed under the MIT License, except for the third-party components listed below which are subject to different license terms. Your integration of RapidJSON into your own projects may require compliance with the MIT License, as well as the other licenses applicable to the third-party components included within RapidJSON. To avoid the problematic JSON license in your own projects, it's sufficient to exclude the bin/jsonchecker/ directory, as it's the only code under the JSON license. +A copy of the MIT License is included in this file. + +Other dependencies and licenses: + +Open Source Software Licensed Under the BSD License: +-------------------------------------------------------------------- + +The msinttypes r29 +Copyright (c) 2006-2013 Alexander Chemeris +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +* Neither the name of copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Open Source Software Licensed Under the JSON License: +-------------------------------------------------------------------- + +json.org +Copyright (c) 2002 JSON.org +All Rights Reserved. + +JSON_checker +Copyright (c) 2002 JSON.org +All Rights Reserved. + + +Terms of the JSON License: +--------------------------------------------------- + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +The Software shall be used for Good, not Evil. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Terms of the MIT License: +-------------------------------------------------------------------- + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/3rdparty/rapidjson/pointer.h b/src/3rdparty/rapidjson/pointer.h index 0206ac1c..063abab9 100644 --- a/src/3rdparty/rapidjson/pointer.h +++ b/src/3rdparty/rapidjson/pointer.h @@ -21,9 +21,7 @@ #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(switch-enum) -#endif - -#ifdef _MSC_VER +#elif defined(_MSC_VER) RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated #endif @@ -165,7 +163,12 @@ public: GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} //! Copy constructor. - GenericPointer(const GenericPointer& rhs, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + GenericPointer(const GenericPointer& rhs) : allocator_(rhs.allocator_), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + *this = rhs; + } + + //! Copy constructor. + GenericPointer(const GenericPointer& rhs, Allocator* allocator) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { *this = rhs; } @@ -197,6 +200,36 @@ public: return *this; } + //! Swap the content of this pointer with an other. + /*! + \param other The pointer to swap with. + \note Constant complexity. + */ + GenericPointer& Swap(GenericPointer& other) RAPIDJSON_NOEXCEPT { + internal::Swap(allocator_, other.allocator_); + internal::Swap(ownAllocator_, other.ownAllocator_); + internal::Swap(nameBuffer_, other.nameBuffer_); + internal::Swap(tokens_, other.tokens_); + internal::Swap(tokenCount_, other.tokenCount_); + internal::Swap(parseErrorOffset_, other.parseErrorOffset_); + internal::Swap(parseErrorCode_, other.parseErrorCode_); + return *this; + } + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.pointer, b.pointer); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericPointer& a, GenericPointer& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } + //@} //!@name Append token @@ -240,7 +273,7 @@ public: template RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >), (GenericPointer)) Append(T* name, Allocator* allocator = 0) const { - return Append(name, StrLen(name), allocator); + return Append(name, internal::StrLen(name), allocator); } #if RAPIDJSON_HAS_STDSTRING @@ -274,7 +307,7 @@ public: else { Ch name[21]; for (size_t i = 0; i <= length; i++) - name[i] = buffer[i]; + name[i] = static_cast(buffer[i]); Token token = { name, length, index }; return Append(token, allocator); } @@ -353,6 +386,33 @@ public: */ bool operator!=(const GenericPointer& rhs) const { return !(*this == rhs); } + //! Less than operator. + /*! + \note Invalid pointers are always greater than valid ones. + */ + bool operator<(const GenericPointer& rhs) const { + if (!IsValid()) + return false; + if (!rhs.IsValid()) + return true; + + if (tokenCount_ != rhs.tokenCount_) + return tokenCount_ < rhs.tokenCount_; + + for (size_t i = 0; i < tokenCount_; i++) { + if (tokens_[i].index != rhs.tokens_[i].index) + return tokens_[i].index < rhs.tokens_[i].index; + + if (tokens_[i].length != rhs.tokens_[i].length) + return tokens_[i].length < rhs.tokens_[i].length; + + if (int cmp = std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch) * tokens_[i].length)) + return cmp < 0; + } + + return false; + } + //@} //!@name Stringify @@ -532,14 +592,14 @@ public: */ ValueType& GetWithDefault(ValueType& root, const ValueType& defaultValue, typename ValueType::AllocatorType& allocator) const { bool alreadyExist; - Value& v = Create(root, allocator, &alreadyExist); + ValueType& v = Create(root, allocator, &alreadyExist); return alreadyExist ? v : v.CopyFrom(defaultValue, allocator); } //! Query a value in a subtree with default null-terminated string. ValueType& GetWithDefault(ValueType& root, const Ch* defaultValue, typename ValueType::AllocatorType& allocator) const { bool alreadyExist; - Value& v = Create(root, allocator, &alreadyExist); + ValueType& v = Create(root, allocator, &alreadyExist); return alreadyExist ? v : v.SetString(defaultValue, allocator); } @@ -547,7 +607,7 @@ public: //! Query a value in a subtree with default std::basic_string. ValueType& GetWithDefault(ValueType& root, const std::basic_string& defaultValue, typename ValueType::AllocatorType& allocator) const { bool alreadyExist; - Value& v = Create(root, allocator, &alreadyExist); + ValueType& v = Create(root, allocator, &alreadyExist); return alreadyExist ? v : v.SetString(defaultValue, allocator); } #endif @@ -758,7 +818,7 @@ private: */ Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) { if (!allocator_) // allocator is independently owned. - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) @@ -806,7 +866,7 @@ private: // Create own allocator if user did not supply. if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); // Count number of '/' as tokenCount tokenCount_ = 0; @@ -1029,8 +1089,8 @@ private: unsigned char u = static_cast(c); static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; os_.Put('%'); - os_.Put(hexDigits[u >> 4]); - os_.Put(hexDigits[u & 15]); + os_.Put(static_cast(hexDigits[u >> 4])); + os_.Put(static_cast(hexDigits[u & 15])); } private: OutputStream& os_; @@ -1347,11 +1407,7 @@ bool EraseValueByPointer(T& root, const CharType(&source)[N]) { RAPIDJSON_NAMESPACE_END -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#ifdef _MSC_VER +#if defined(__clang__) || defined(_MSC_VER) RAPIDJSON_DIAG_POP #endif diff --git a/src/3rdparty/rapidjson/prettywriter.h b/src/3rdparty/rapidjson/prettywriter.h index 0dcb0fee..45afb694 100644 --- a/src/3rdparty/rapidjson/prettywriter.h +++ b/src/3rdparty/rapidjson/prettywriter.h @@ -22,6 +22,11 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(effc++) #endif +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + RAPIDJSON_NAMESPACE_BEGIN //! Combination of PrettyWriter format flags. @@ -34,7 +39,7 @@ enum PrettyFormatOptions { //! Writer with indentation and spacing. /*! - \tparam OutputStream Type of ouptut os. + \tparam OutputStream Type of output os. \tparam SourceEncoding Encoding of source string. \tparam TargetEncoding Encoding of output stream. \tparam StackAllocator Type of allocator for allocating memory of stack. @@ -42,7 +47,7 @@ enum PrettyFormatOptions { template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> class PrettyWriter : public Writer { public: - typedef Writer Base; + typedef Writer Base; typedef typename Base::Ch Ch; //! Constructor @@ -57,6 +62,11 @@ public: explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + PrettyWriter(PrettyWriter&& rhs) : + Base(std::forward(rhs)), indentChar_(rhs.indentChar_), indentCharCount_(rhs.indentCharCount_), formatOptions_(rhs.formatOptions_) {} +#endif + //! Set custom indentation. /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). \param indentCharCount Number of indent characters for each indentation level. @@ -82,24 +92,26 @@ public: */ //@{ - bool Null() { PrettyPrefix(kNullType); return Base::WriteNull(); } - bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::WriteBool(b); } - bool Int(int i) { PrettyPrefix(kNumberType); return Base::WriteInt(i); } - bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::WriteUint(u); } - bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::WriteInt64(i64); } - bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::WriteUint64(u64); } - bool Double(double d) { PrettyPrefix(kNumberType); return Base::WriteDouble(d); } + bool Null() { PrettyPrefix(kNullType); return Base::EndValue(Base::WriteNull()); } + bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::EndValue(Base::WriteBool(b)); } + bool Int(int i) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteInt(i)); } + bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteUint(u)); } + bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteInt64(i64)); } + bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteUint64(u64)); } + bool Double(double d) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteDouble(d)); } bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); (void)copy; PrettyPrefix(kNumberType); - return Base::WriteString(str, length); + return Base::EndValue(Base::WriteString(str, length)); } bool String(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); (void)copy; PrettyPrefix(kStringType); - return Base::WriteString(str, length); + return Base::EndValue(Base::WriteString(str, length)); } #if RAPIDJSON_HAS_STDSTRING @@ -124,19 +136,21 @@ public: bool EndObject(SizeType memberCount = 0) { (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); - RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); // not inside an Object + RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); // currently inside an Array, not Object + RAPIDJSON_ASSERT(0 == Base::level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; if (!empty) { Base::os_->Put('\n'); WriteIndent(); } - bool ret = Base::WriteEndObject(); + bool ret = Base::EndValue(Base::WriteEndObject()); (void)ret; RAPIDJSON_ASSERT(ret == true); if (Base::level_stack_.Empty()) // end of json text - Base::os_->Flush(); + Base::Flush(); return true; } @@ -156,11 +170,11 @@ public: Base::os_->Put('\n'); WriteIndent(); } - bool ret = Base::WriteEndArray(); + bool ret = Base::EndValue(Base::WriteEndArray()); (void)ret; RAPIDJSON_ASSERT(ret == true); if (Base::level_stack_.Empty()) // end of json text - Base::os_->Flush(); + Base::Flush(); return true; } @@ -184,7 +198,11 @@ public: \param type Type of the root of json. \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. */ - bool RawValue(const Ch* json, size_t length, Type type) { PrettyPrefix(type); return Base::WriteRawValue(json, length); } + bool RawValue(const Ch* json, size_t length, Type type) { + RAPIDJSON_ASSERT(json != 0); + PrettyPrefix(type); + return Base::EndValue(Base::WriteRawValue(json, length)); + } protected: void PrettyPrefix(Type type) { @@ -233,7 +251,7 @@ protected: void WriteIndent() { size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; - PutN(*Base::os_, static_cast(indentChar_), count); + PutN(*Base::os_, static_cast(indentChar_), count); } Ch indentChar_; @@ -248,6 +266,10 @@ private: RAPIDJSON_NAMESPACE_END +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + #ifdef __GNUC__ RAPIDJSON_DIAG_POP #endif diff --git a/src/3rdparty/rapidjson/rapidjson.h b/src/3rdparty/rapidjson/rapidjson.h index 2ef9bc56..549936ff 100644 --- a/src/3rdparty/rapidjson/rapidjson.h +++ b/src/3rdparty/rapidjson/rapidjson.h @@ -26,7 +26,7 @@ Some RapidJSON features are configurable to adapt the library to a wide variety of platforms, environments and usage scenarios. Most of the - features can be configured in terms of overriden or predefined + features can be configured in terms of overridden or predefined preprocessor macros at compile-time. Some additional customization is available in the \ref RAPIDJSON_ERRORS APIs. @@ -49,6 +49,11 @@ // token stringification #define RAPIDJSON_STRINGIFY(x) RAPIDJSON_DO_STRINGIFY(x) #define RAPIDJSON_DO_STRINGIFY(x) #x + +// token concatenation +#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) +#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) +#define RAPIDJSON_DO_JOIN2(X, Y) X##Y //!@endcond /*! \def RAPIDJSON_MAJOR_VERSION @@ -214,7 +219,7 @@ # elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ # define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN # else -# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# error Unknown machine endianness detected. User needs to define RAPIDJSON_ENDIAN. # endif // __BYTE_ORDER__ // Detect with GLIBC's endian.h # elif defined(__GLIBC__) @@ -224,7 +229,7 @@ # elif (__BYTE_ORDER == __BIG_ENDIAN) # define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN # else -# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# error Unknown machine endianness detected. User needs to define RAPIDJSON_ENDIAN. # endif // __GLIBC__ // Detect with _LITTLE_ENDIAN and _BIG_ENDIAN macro # elif defined(_LITTLE_ENDIAN) && !defined(_BIG_ENDIAN) @@ -236,12 +241,12 @@ # define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN # elif defined(__i386__) || defined(__alpha__) || defined(__ia64) || defined(__ia64__) || defined(_M_IX86) || defined(_M_IA64) || defined(_M_ALPHA) || defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || defined(__bfin__) # define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN -# elif defined(_MSC_VER) && defined(_M_ARM) +# elif defined(_MSC_VER) && (defined(_M_ARM) || defined(_M_ARM64)) # define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN # elif defined(RAPIDJSON_DOXYGEN_RUNNING) # define RAPIDJSON_ENDIAN # else -# error Unknown machine endianess detected. User needs to define RAPIDJSON_ENDIAN. +# error Unknown machine endianness detected. User needs to define RAPIDJSON_ENDIAN. # endif #endif // RAPIDJSON_ENDIAN @@ -264,16 +269,11 @@ /*! \ingroup RAPIDJSON_CONFIG \param x pointer to align - Some machines require strict data alignment. Currently the default uses 4 bytes - alignment on 32-bit platforms and 8 bytes alignment for 64-bit platforms. + Some machines require strict data alignment. The default is 8 bytes. User can customize by defining the RAPIDJSON_ALIGN function macro. */ #ifndef RAPIDJSON_ALIGN -#if RAPIDJSON_64BIT == 1 -#define RAPIDJSON_ALIGN(x) (((x) + static_cast(7u)) & ~static_cast(7u)) -#else -#define RAPIDJSON_ALIGN(x) (((x) + 3u) & ~3u) -#endif +#define RAPIDJSON_ALIGN(x) (((x) + static_cast(7u)) & ~static_cast(7u)) #endif /////////////////////////////////////////////////////////////////////////////// @@ -320,17 +320,17 @@ #endif /////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_SIMD +// RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_NEON/RAPIDJSON_SIMD /*! \def RAPIDJSON_SIMD \ingroup RAPIDJSON_CONFIG - \brief Enable SSE2/SSE4.2 optimization. + \brief Enable SSE2/SSE4.2/Neon optimization. RapidJSON supports optimized implementations for some parsing operations - based on the SSE2 or SSE4.2 SIMD extensions on modern Intel-compatible - processors. + based on the SSE2, SSE4.2 or NEon SIMD extensions on modern Intel + or ARM compatible processors. - To enable these optimizations, two different symbols can be defined; + To enable these optimizations, three different symbols can be defined; \code // Enable SSE2 optimization. #define RAPIDJSON_SSE2 @@ -339,13 +339,17 @@ #define RAPIDJSON_SSE42 \endcode - \c RAPIDJSON_SSE42 takes precedence, if both are defined. + // Enable ARM Neon optimization. + #define RAPIDJSON_NEON + \endcode + + \c RAPIDJSON_SSE42 takes precedence over SSE2, if both are defined. If any of these symbols is defined, RapidJSON defines the macro \c RAPIDJSON_SIMD to indicate the availability of the optimized code. */ #if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) \ - || defined(RAPIDJSON_DOXYGEN_RUNNING) + || defined(RAPIDJSON_NEON) || defined(RAPIDJSON_DOXYGEN_RUNNING) #define RAPIDJSON_SIMD #endif @@ -398,13 +402,22 @@ RAPIDJSON_NAMESPACE_END \ref RAPIDJSON_ERRORS APIs. */ #ifndef RAPIDJSON_ASSERT -#define RAPIDJSON_ASSERT(x) +#include +#define RAPIDJSON_ASSERT(x) assert(x) #endif // RAPIDJSON_ASSERT /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_STATIC_ASSERT -// Adopt from boost +// Prefer C++11 static_assert, if available +#ifndef RAPIDJSON_STATIC_ASSERT +#if __cplusplus >= 201103L || ( defined(_MSC_VER) && _MSC_VER >= 1800 ) +#define RAPIDJSON_STATIC_ASSERT(x) \ + static_assert(x, RAPIDJSON_STRINGIFY(x)) +#endif // C++11 +#endif // RAPIDJSON_STATIC_ASSERT + +// Adopt C++03 implementation from boost #ifndef RAPIDJSON_STATIC_ASSERT #ifndef __clang__ //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN @@ -412,14 +425,10 @@ RAPIDJSON_NAMESPACE_END RAPIDJSON_NAMESPACE_BEGIN template struct STATIC_ASSERTION_FAILURE; template <> struct STATIC_ASSERTION_FAILURE { enum { value = 1 }; }; -template struct StaticAssertTest {}; +template struct StaticAssertTest {}; RAPIDJSON_NAMESPACE_END -#define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) -#define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) -#define RAPIDJSON_DO_JOIN2(X, Y) X##Y - -#if defined(__GNUC__) +#if defined(__GNUC__) || defined(__clang__) #define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE __attribute__((unused)) #else #define RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE @@ -437,7 +446,7 @@ RAPIDJSON_NAMESPACE_END typedef ::RAPIDJSON_NAMESPACE::StaticAssertTest< \ sizeof(::RAPIDJSON_NAMESPACE::STATIC_ASSERTION_FAILURE)> \ RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE -#endif +#endif // RAPIDJSON_STATIC_ASSERT /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_LIKELY, RAPIDJSON_UNLIKELY @@ -529,13 +538,14 @@ RAPIDJSON_NAMESPACE_END #ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS #if defined(__clang__) #if __has_feature(cxx_rvalue_references) && \ - (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) + (defined(_MSC_VER) || defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) #define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 #else #define RAPIDJSON_HAS_CXX11_RVALUE_REFS 0 #endif #elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ - (defined(_MSC_VER) && _MSC_VER >= 1600) + (defined(_MSC_VER) && _MSC_VER >= 1600) || \ + (defined(__SUNPRO_CC) && __SUNPRO_CC >= 0x5140 && defined(__GXX_EXPERIMENTAL_CXX0X__)) #define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 #else @@ -546,8 +556,9 @@ RAPIDJSON_NAMESPACE_END #ifndef RAPIDJSON_HAS_CXX11_NOEXCEPT #if defined(__clang__) #define RAPIDJSON_HAS_CXX11_NOEXCEPT __has_feature(cxx_noexcept) -#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) -// (defined(_MSC_VER) && _MSC_VER >= ????) // not yet supported +#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1900) || \ + (defined(__SUNPRO_CC) && __SUNPRO_CC >= 0x5140 && defined(__GXX_EXPERIMENTAL_CXX0X__)) #define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 #else #define RAPIDJSON_HAS_CXX11_NOEXCEPT 0 @@ -561,14 +572,19 @@ RAPIDJSON_NAMESPACE_END // no automatic detection, yet #ifndef RAPIDJSON_HAS_CXX11_TYPETRAITS +#if (defined(_MSC_VER) && _MSC_VER >= 1700) +#define RAPIDJSON_HAS_CXX11_TYPETRAITS 1 +#else #define RAPIDJSON_HAS_CXX11_TYPETRAITS 0 #endif +#endif #ifndef RAPIDJSON_HAS_CXX11_RANGE_FOR #if defined(__clang__) #define RAPIDJSON_HAS_CXX11_RANGE_FOR __has_feature(cxx_range_for) -#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,3,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ - (defined(_MSC_VER) && _MSC_VER >= 1700) +#elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1700) || \ + (defined(__SUNPRO_CC) && __SUNPRO_CC >= 0x5140 && defined(__GXX_EXPERIMENTAL_CXX0X__)) #define RAPIDJSON_HAS_CXX11_RANGE_FOR 1 #else #define RAPIDJSON_HAS_CXX11_RANGE_FOR 0 @@ -577,12 +593,38 @@ RAPIDJSON_NAMESPACE_END //!@endcond +//! Assertion (in non-throwing contexts). + /*! \ingroup RAPIDJSON_CONFIG + Some functions provide a \c noexcept guarantee, if the compiler supports it. + In these cases, the \ref RAPIDJSON_ASSERT macro cannot be overridden to + throw an exception. This macro adds a separate customization point for + such cases. + + Defaults to C \c assert() (as \ref RAPIDJSON_ASSERT), if \c noexcept is + supported, and to \ref RAPIDJSON_ASSERT otherwise. + */ + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_NOEXCEPT_ASSERT + +#ifndef RAPIDJSON_NOEXCEPT_ASSERT +#ifdef RAPIDJSON_ASSERT_THROWS +#if RAPIDJSON_HAS_CXX11_NOEXCEPT +#define RAPIDJSON_NOEXCEPT_ASSERT(x) +#else +#define RAPIDJSON_NOEXCEPT_ASSERT(x) RAPIDJSON_ASSERT(x) +#endif // RAPIDJSON_HAS_CXX11_NOEXCEPT +#else +#define RAPIDJSON_NOEXCEPT_ASSERT(x) RAPIDJSON_ASSERT(x) +#endif // RAPIDJSON_ASSERT_THROWS +#endif // RAPIDJSON_NOEXCEPT_ASSERT + /////////////////////////////////////////////////////////////////////////////// // new/delete #ifndef RAPIDJSON_NEW ///! customization point for global \c new -#define RAPIDJSON_NEW(x) new x +#define RAPIDJSON_NEW(TypeName) new TypeName #endif #ifndef RAPIDJSON_DELETE ///! customization point for global \c delete diff --git a/src/3rdparty/rapidjson/reader.h b/src/3rdparty/rapidjson/reader.h index 303aac2e..44a6bcd3 100644 --- a/src/3rdparty/rapidjson/reader.h +++ b/src/3rdparty/rapidjson/reader.h @@ -33,12 +33,8 @@ #include #elif defined(RAPIDJSON_SSE2) #include -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant -RAPIDJSON_DIAG_OFF(4702) // unreachable code +#elif defined(RAPIDJSON_NEON) +#include #endif #ifdef __clang__ @@ -46,6 +42,10 @@ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(old-style-cast) RAPIDJSON_DIAG_OFF(padded) RAPIDJSON_DIAG_OFF(switch-enum) +#elif defined(_MSC_VER) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +RAPIDJSON_DIAG_OFF(4702) // unreachable code #endif #ifdef __GNUC__ @@ -136,7 +136,7 @@ RAPIDJSON_NAMESPACE_BEGIN User can define this as any \c ParseFlag combinations. */ #ifndef RAPIDJSON_PARSE_DEFAULT_FLAGS -#define RAPIDJSON_PARSE_DEFAULT_FLAGS kParseCommentsFlag | kParseTrailingCommasFlag +#define RAPIDJSON_PARSE_DEFAULT_FLAGS kParseNoFlags #endif //! Combination of parseFlags @@ -299,16 +299,9 @@ inline const char *SkipWhitespace_SIMD(const char* p) { for (;; p += 16) { const __m128i s = _mm_load_si128(reinterpret_cast(p)); - const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); - if (r != 0) { // some of characters is non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; -#else - return p + __builtin_ffs(r) - 1; -#endif - } + const int r = _mm_cmpistri(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT | _SIDD_NEGATIVE_POLARITY); + if (r != 16) // some of characters is non-whitespace + return p + r; } } @@ -325,16 +318,9 @@ inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { for (; p <= end - 16; p += 16) { const __m128i s = _mm_loadu_si128(reinterpret_cast(p)); - const int r = _mm_cvtsi128_si32(_mm_cmpistrm(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK | _SIDD_NEGATIVE_POLARITY)); - if (r != 0) { // some of characters is non-whitespace -#ifdef _MSC_VER // Find the index of first non-whitespace - unsigned long offset; - _BitScanForward(&offset, r); - return p + offset; -#else - return p + __builtin_ffs(r) - 1; -#endif - } + const int r = _mm_cmpistri(w, s, _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT | _SIDD_NEGATIVE_POLARITY); + if (r != 16) // some of characters is non-whitespace + return p + r; } return SkipWhitespace(p, end); @@ -425,7 +411,92 @@ inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { return SkipWhitespace(p, end); } -#endif // RAPIDJSON_SSE2 +#elif defined(RAPIDJSON_NEON) + +//! Skip whitespace with ARM Neon instructions, testing 16 8-byte characters at once. +inline const char *SkipWhitespace_SIMD(const char* p) { + // Fast return for single non-whitespace + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + // 16-byte align to the next boundary + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') + ++p; + else + return p; + + const uint8x16_t w0 = vmovq_n_u8(' '); + const uint8x16_t w1 = vmovq_n_u8('\n'); + const uint8x16_t w2 = vmovq_n_u8('\r'); + const uint8x16_t w3 = vmovq_n_u8('\t'); + + for (;; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, w0); + x = vorrq_u8(x, vceqq_u8(s, w1)); + x = vorrq_u8(x, vceqq_u8(s, w2)); + x = vorrq_u8(x, vceqq_u8(s, w3)); + + x = vmvnq_u8(x); // Negate + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + if (low == 0) { + if (high != 0) { + int lz =__builtin_clzll(high);; + return p + 8 + (lz >> 3); + } + } else { + int lz = __builtin_clzll(low);; + return p + (lz >> 3); + } + } +} + +inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { + // Fast return for single non-whitespace + if (p != end && (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')) + ++p; + else + return p; + + const uint8x16_t w0 = vmovq_n_u8(' '); + const uint8x16_t w1 = vmovq_n_u8('\n'); + const uint8x16_t w2 = vmovq_n_u8('\r'); + const uint8x16_t w3 = vmovq_n_u8('\t'); + + for (; p <= end - 16; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, w0); + x = vorrq_u8(x, vceqq_u8(s, w1)); + x = vorrq_u8(x, vceqq_u8(s, w2)); + x = vorrq_u8(x, vceqq_u8(s, w3)); + + x = vmvnq_u8(x); // Negate + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + if (low == 0) { + if (high != 0) { + int lz = __builtin_clzll(high); + return p + 8 + (lz >> 3); + } + } else { + int lz = __builtin_clzll(low); + return p + (lz >> 3); + } + } + + return SkipWhitespace(p, end); +} + +#endif // RAPIDJSON_NEON #ifdef RAPIDJSON_SIMD //! Template function specialization for InsituStringStream @@ -471,7 +542,8 @@ public: /*! \param stackAllocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing) \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) */ - GenericReader(StackAllocator* stackAllocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(stackAllocator, stackCapacity), parseResult_() {} + GenericReader(StackAllocator* stackAllocator = 0, size_t stackCapacity = kDefaultStackCapacity) : + stack_(stackAllocator, stackCapacity), parseResult_(), state_(IterativeParsingStartState) {} //! Parse JSON text. /*! \tparam parseFlags Combination of \ref ParseFlag. @@ -527,7 +599,84 @@ public: return Parse(is, handler); } - //! Whether a parse error has occured in the last parsing. + //! Initialize JSON text token-by-token parsing + /*! + */ + void IterativeParseInit() { + parseResult_.Clear(); + state_ = IterativeParsingStartState; + } + + //! Parse one token from JSON text + /*! \tparam InputStream Type of input stream, implementing Stream concept + \tparam Handler Type of handler, implementing Handler concept. + \param is Input stream to be parsed. + \param handler The handler to receive events. + \return Whether the parsing is successful. + */ + template + bool IterativeParseNext(InputStream& is, Handler& handler) { + while (RAPIDJSON_LIKELY(is.Peek() != '\0')) { + SkipWhitespaceAndComments(is); + + Token t = Tokenize(is.Peek()); + IterativeParsingState n = Predict(state_, t); + IterativeParsingState d = Transit(state_, t, n, is, handler); + + // If we've finished or hit an error... + if (RAPIDJSON_UNLIKELY(IsIterativeParsingCompleteState(d))) { + // Report errors. + if (d == IterativeParsingErrorState) { + HandleError(state_, is); + return false; + } + + // Transition to the finish state. + RAPIDJSON_ASSERT(d == IterativeParsingFinishState); + state_ = d; + + // If StopWhenDone is not set... + if (!(parseFlags & kParseStopWhenDoneFlag)) { + // ... and extra non-whitespace data is found... + SkipWhitespaceAndComments(is); + if (is.Peek() != '\0') { + // ... this is considered an error. + HandleError(state_, is); + return false; + } + } + + // Success! We are done! + return true; + } + + // Transition to the new state. + state_ = d; + + // If we parsed anything other than a delimiter, we invoked the handler, so we can return true now. + if (!IsIterativeParsingDelimiterState(n)) + return true; + } + + // We reached the end of file. + stack_.Clear(); + + if (state_ != IterativeParsingFinishState) { + HandleError(state_, is); + return false; + } + + return true; + } + + //! Check if token-by-token parsing JSON text is complete + /*! \return Whether the JSON has been fully decoded. + */ + RAPIDJSON_FORCEINLINE bool IterativeParseComplete() const { + return IsIterativeParsingCompleteState(state_); + } + + //! Whether a parse error has occurred in the last parsing. bool HasParseError() const { return parseResult_.IsError(); } //! Get the \ref ParseErrorCode of last parsing. @@ -575,7 +724,7 @@ private: } } else if (RAPIDJSON_LIKELY(Consume(is, '/'))) - while (is.Peek() != '\0' && is.Take() != '\n'); + while (is.Peek() != '\0' && is.Take() != '\n') {} else RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); @@ -750,7 +899,7 @@ private: return false; } - // Helper function to parse four hexidecimal digits in \uXXXX in ParseString(). + // Helper function to parse four hexadecimal digits in \uXXXX in ParseString(). template unsigned ParseHex4(InputStream& is, size_t escapeOffset) { unsigned codepoint = 0; @@ -857,7 +1006,7 @@ private: Ch c = is.Peek(); if (RAPIDJSON_UNLIKELY(c == '\\')) { // Escape - size_t escapeOffset = is.Tell(); // For invalid escaping, report the inital '\\' as error offset + size_t escapeOffset = is.Tell(); // For invalid escaping, report the initial '\\' as error offset is.Take(); Ch e = is.Peek(); if ((sizeof(Ch) == 1 || unsigned(e) < 256) && RAPIDJSON_LIKELY(escape[static_cast(e)])) { @@ -892,7 +1041,7 @@ private: if (c == '\0') RAPIDJSON_PARSE_ERROR(kParseErrorStringMissQuotationMark, is.Tell()); else - RAPIDJSON_PARSE_ERROR(kParseErrorStringEscapeInvalid, is.Tell()); + RAPIDJSON_PARSE_ERROR(kParseErrorStringInvalidEncoding, is.Tell()); } else { size_t offset = is.Tell(); @@ -927,7 +1076,7 @@ private: // The rest of string using SIMD static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); @@ -936,7 +1085,7 @@ private: const __m128i s = _mm_load_si128(reinterpret_cast(p)); const __m128i t1 = _mm_cmpeq_epi8(s, dq); const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); unsigned short r = static_cast(_mm_movemask_epi8(x)); if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped @@ -948,11 +1097,13 @@ private: #else length = static_cast(__builtin_ffs(r) - 1); #endif - char* q = reinterpret_cast(os.Push(length)); - for (size_t i = 0; i < length; i++) - q[i] = p[i]; + if (length != 0) { + char* q = reinterpret_cast(os.Push(length)); + for (size_t i = 0; i < length; i++) + q[i] = p[i]; - p += length; + p += length; + } break; } _mm_storeu_si128(reinterpret_cast<__m128i *>(os.Push(16)), s); @@ -988,7 +1139,7 @@ private: // The rest of string using SIMD static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); @@ -997,7 +1148,7 @@ private: const __m128i s = _mm_load_si128(reinterpret_cast(p)); const __m128i t1 = _mm_cmpeq_epi8(s, dq); const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); unsigned short r = static_cast(_mm_movemask_epi8(x)); if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped @@ -1036,7 +1187,7 @@ private: // The rest of string using SIMD static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); @@ -1045,7 +1196,7 @@ private: const __m128i s = _mm_load_si128(reinterpret_cast(p)); const __m128i t1 = _mm_cmpeq_epi8(s, dq); const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); unsigned short r = static_cast(_mm_movemask_epi8(x)); if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped @@ -1064,7 +1215,180 @@ private: is.src_ = is.dst_ = p; } -#endif +#elif defined(RAPIDJSON_NEON) + // StringStream -> StackStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(StringStream& is, StackStream& os) { + const char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + return; + } + else + os.Put(*p++); + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (;; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + SizeType length = 0; + bool escaped = false; + if (low == 0) { + if (high != 0) { + unsigned lz = (unsigned)__builtin_clzll(high);; + length = 8 + (lz >> 3); + escaped = true; + } + } else { + unsigned lz = (unsigned)__builtin_clzll(low);; + length = lz >> 3; + escaped = true; + } + if (RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped + if (length != 0) { + char* q = reinterpret_cast(os.Push(length)); + for (size_t i = 0; i < length; i++) + q[i] = p[i]; + + p += length; + } + break; + } + vst1q_u8(reinterpret_cast(os.Push(16)), s); + } + + is.src_ = p; + } + + // InsituStringStream -> InsituStringStream + static RAPIDJSON_FORCEINLINE void ScanCopyUnescapedString(InsituStringStream& is, InsituStringStream& os) { + RAPIDJSON_ASSERT(&is == &os); + (void)os; + + if (is.src_ == is.dst_) { + SkipUnescapedString(is); + return; + } + + char* p = is.src_; + char *q = is.dst_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + while (p != nextAligned) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = p; + is.dst_ = q; + return; + } + else + *q++ = *p++; + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (;; p += 16, q += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + SizeType length = 0; + bool escaped = false; + if (low == 0) { + if (high != 0) { + unsigned lz = (unsigned)__builtin_clzll(high); + length = 8 + (lz >> 3); + escaped = true; + } + } else { + unsigned lz = (unsigned)__builtin_clzll(low); + length = lz >> 3; + escaped = true; + } + if (RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped + for (const char* pend = p + length; p != pend; ) { + *q++ = *p++; + } + break; + } + vst1q_u8(reinterpret_cast(q), s); + } + + is.src_ = p; + is.dst_ = q; + } + + // When read/write pointers are the same for insitu stream, just skip unescaped characters + static RAPIDJSON_FORCEINLINE void SkipUnescapedString(InsituStringStream& is) { + RAPIDJSON_ASSERT(is.src_ == is.dst_); + char* p = is.src_; + + // Scan one by one until alignment (unaligned load may cross page boundary and cause crash) + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + for (; p != nextAligned; p++) + if (RAPIDJSON_UNLIKELY(*p == '\"') || RAPIDJSON_UNLIKELY(*p == '\\') || RAPIDJSON_UNLIKELY(static_cast(*p) < 0x20)) { + is.src_ = is.dst_ = p; + return; + } + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (;; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + if (low == 0) { + if (high != 0) { + int lz = __builtin_clzll(high); + p += 8 + (lz >> 3); + break; + } + } else { + int lz = __builtin_clzll(low); + p += lz >> 3; + break; + } + } + + is.src_ = is.dst_ = p; + } +#endif // RAPIDJSON_NEON template class NumberStream; @@ -1075,7 +1399,6 @@ private: typedef typename InputStream::Ch Ch; NumberStream(GenericReader& reader, InputStream& s) : is(s) { (void)reader; } - ~NumberStream() {} RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } @@ -1097,7 +1420,6 @@ private: typedef NumberStream Base; public: NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} - ~NumberStream() {} RAPIDJSON_FORCEINLINE Ch TakePush() { stackStream.Put(static_cast(Base::is.Peek())); @@ -1124,7 +1446,6 @@ private: typedef NumberStream Base; public: NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} - ~NumberStream() {} RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } }; @@ -1185,18 +1506,27 @@ private: } // Parse NaN or Infinity here else if ((parseFlags & kParseNanAndInfFlag) && RAPIDJSON_LIKELY((s.Peek() == 'I' || s.Peek() == 'N'))) { - useNanOrInf = true; - if (RAPIDJSON_LIKELY(Consume(s, 'N') && Consume(s, 'a') && Consume(s, 'N'))) { - d = std::numeric_limits::quiet_NaN(); + if (Consume(s, 'N')) { + if (Consume(s, 'a') && Consume(s, 'N')) { + d = std::numeric_limits::quiet_NaN(); + useNanOrInf = true; + } } - else if (RAPIDJSON_LIKELY(Consume(s, 'I') && Consume(s, 'n') && Consume(s, 'f'))) { - d = (minus ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); - if (RAPIDJSON_UNLIKELY(s.Peek() == 'i' && !(Consume(s, 'i') && Consume(s, 'n') - && Consume(s, 'i') && Consume(s, 't') && Consume(s, 'y')))) - RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + else if (RAPIDJSON_LIKELY(Consume(s, 'I'))) { + if (Consume(s, 'n') && Consume(s, 'f')) { + d = (minus ? -std::numeric_limits::infinity() : std::numeric_limits::infinity()); + useNanOrInf = true; + + if (RAPIDJSON_UNLIKELY(s.Peek() == 'i' && !(Consume(s, 'i') && Consume(s, 'n') + && Consume(s, 'i') && Consume(s, 't') && Consume(s, 'y')))) { + RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } + } } - else + + if (RAPIDJSON_UNLIKELY(!useNanOrInf)) { RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); + } } else RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, s.Tell()); @@ -1231,8 +1561,6 @@ private: // Force double for big integer if (useDouble) { while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { - if (RAPIDJSON_UNLIKELY(d >= 1.7976931348623157e307)) // DBL_MAX / 10.0 - RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); d = d * 10 + (s.TakePush() - '0'); } } @@ -1302,9 +1630,18 @@ private: if (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { exp = static_cast(s.Take() - '0'); if (expMinus) { + // (exp + expFrac) must not underflow int => we're detecting when -exp gets + // dangerously close to INT_MIN (a pessimistic next digit 9 would push it into + // underflow territory): + // + // -(exp * 10 + 9) + expFrac >= INT_MIN + // <=> exp <= (expFrac - INT_MIN - 9) / 10 + RAPIDJSON_ASSERT(expFrac <= 0); + int maxExp = (expFrac + 2147483639) / 10; + while (RAPIDJSON_LIKELY(s.Peek() >= '0' && s.Peek() <= '9')) { exp = exp * 10 + static_cast(s.Take() - '0'); - if (exp >= 214748364) { // Issue #313: prevent overflow exponent + if (RAPIDJSON_UNLIKELY(exp > maxExp)) { while (RAPIDJSON_UNLIKELY(s.Peek() >= '0' && s.Peek() <= '9')) // Consume the rest of exponent s.Take(); } @@ -1363,6 +1700,13 @@ private: else d = internal::StrtodNormalPrecision(d, p); + // Use > max, instead of == inf, to fix bogus warning -Wfloat-equal + if (d > (std::numeric_limits::max)()) { + // Overflow + // TODO: internal::StrtodX should report overflow (or underflow) + RAPIDJSON_PARSE_ERROR(kParseErrorNumberTooBig, startOffset); + } + cont = handler.Double(minus ? -d : d); } else if (useNanOrInf) { @@ -1408,29 +1752,31 @@ private: // States enum IterativeParsingState { - IterativeParsingStartState = 0, - IterativeParsingFinishState, - IterativeParsingErrorState, + IterativeParsingFinishState = 0, // sink states at top + IterativeParsingErrorState, // sink states at top + IterativeParsingStartState, // Object states IterativeParsingObjectInitialState, IterativeParsingMemberKeyState, - IterativeParsingKeyValueDelimiterState, IterativeParsingMemberValueState, - IterativeParsingMemberDelimiterState, IterativeParsingObjectFinishState, // Array states IterativeParsingArrayInitialState, IterativeParsingElementState, - IterativeParsingElementDelimiterState, IterativeParsingArrayFinishState, // Single value state - IterativeParsingValueState - }; + IterativeParsingValueState, - enum { cIterativeParsingStateCount = IterativeParsingValueState + 1 }; + // Delimiter states (at bottom) + IterativeParsingElementDelimiterState, + IterativeParsingMemberDelimiterState, + IterativeParsingKeyValueDelimiterState, + + cIterativeParsingStateCount + }; // Tokens enum Token { @@ -1452,7 +1798,7 @@ private: kTokenCount }; - RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) { + RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) const { //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #define N NumberToken @@ -1479,9 +1825,21 @@ private: return NumberToken; } - RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) { + RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) const { // current state x one lookahead token -> new state static const char G[cIterativeParsingStateCount][kTokenCount] = { + // Finish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Error(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, // Start { IterativeParsingArrayInitialState, // Left bracket @@ -1496,18 +1854,6 @@ private: IterativeParsingValueState, // Null IterativeParsingValueState // Number }, - // Finish(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, - // Error(sink state) - { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - }, // ObjectInitial { IterativeParsingErrorState, // Left bracket @@ -1536,20 +1882,6 @@ private: IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, - // KeyValueDelimiter - { - IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) - IterativeParsingErrorState, // Right bracket - IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) - IterativeParsingErrorState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberValueState, // String - IterativeParsingMemberValueState, // False - IterativeParsingMemberValueState, // True - IterativeParsingMemberValueState, // Null - IterativeParsingMemberValueState // Number - }, // MemberValue { IterativeParsingErrorState, // Left bracket @@ -1564,20 +1896,6 @@ private: IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, - // MemberDelimiter - { - IterativeParsingErrorState, // Left bracket - IterativeParsingErrorState, // Right bracket - IterativeParsingErrorState, // Left curly bracket - IterativeParsingObjectFinishState, // Right curly bracket - IterativeParsingErrorState, // Comma - IterativeParsingErrorState, // Colon - IterativeParsingMemberKeyState, // String - IterativeParsingErrorState, // False - IterativeParsingErrorState, // True - IterativeParsingErrorState, // Null - IterativeParsingErrorState // Number - }, // ObjectFinish(sink state) { IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, @@ -1612,6 +1930,18 @@ private: IterativeParsingErrorState, // Null IterativeParsingErrorState // Number }, + // ArrayFinish(sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, + // Single Value (sink state) + { + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, + IterativeParsingErrorState + }, // ElementDelimiter { IterativeParsingArrayInitialState, // Left bracket(push Element state) @@ -1626,18 +1956,34 @@ private: IterativeParsingElementState, // Null IterativeParsingElementState // Number }, - // ArrayFinish(sink state) + // MemberDelimiter { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState + IterativeParsingErrorState, // Left bracket + IterativeParsingErrorState, // Right bracket + IterativeParsingErrorState, // Left curly bracket + IterativeParsingObjectFinishState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberKeyState, // String + IterativeParsingErrorState, // False + IterativeParsingErrorState, // True + IterativeParsingErrorState, // Null + IterativeParsingErrorState // Number }, - // Single Value (sink state) + // KeyValueDelimiter { - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, IterativeParsingErrorState, - IterativeParsingErrorState - } + IterativeParsingArrayInitialState, // Left bracket(push MemberValue state) + IterativeParsingErrorState, // Right bracket + IterativeParsingObjectInitialState, // Left curly bracket(push MemberValue state) + IterativeParsingErrorState, // Right curly bracket + IterativeParsingErrorState, // Comma + IterativeParsingErrorState, // Colon + IterativeParsingMemberValueState, // String + IterativeParsingMemberValueState, // False + IterativeParsingMemberValueState, // True + IterativeParsingMemberValueState, // Null + IterativeParsingMemberValueState // Number + }, }; // End of G return static_cast(G[state][token]); @@ -1818,6 +2164,14 @@ private: } } + RAPIDJSON_FORCEINLINE bool IsIterativeParsingDelimiterState(IterativeParsingState s) const { + return s >= IterativeParsingElementDelimiterState; + } + + RAPIDJSON_FORCEINLINE bool IsIterativeParsingCompleteState(IterativeParsingState s) const { + return s <= IterativeParsingErrorState; + } + template ParseResult IterativeParse(InputStream& is, Handler& handler) { parseResult_.Clear(); @@ -1856,6 +2210,7 @@ private: static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. internal::Stack stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. ParseResult parseResult_; + IterativeParsingState state_; }; // class GenericReader //! Reader with UTF8 encoding and default allocator. @@ -1863,7 +2218,7 @@ typedef GenericReader, UTF8<> > Reader; RAPIDJSON_NAMESPACE_END -#ifdef __clang__ +#if defined(__clang__) || defined(_MSC_VER) RAPIDJSON_DIAG_POP #endif @@ -1872,8 +2227,4 @@ RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_POP #endif -#ifdef _MSC_VER -RAPIDJSON_DIAG_POP -#endif - #endif // RAPIDJSON_READER_H_ diff --git a/src/3rdparty/rapidjson/schema.h b/src/3rdparty/rapidjson/schema.h index b182aa27..26ae9474 100644 --- a/src/3rdparty/rapidjson/schema.h +++ b/src/3rdparty/rapidjson/schema.h @@ -17,6 +17,7 @@ #include "document.h" #include "pointer.h" +#include "stringbuffer.h" #include // abs, floor #if !defined(RAPIDJSON_SCHEMA_USE_INTERNALREGEX) @@ -25,7 +26,7 @@ #define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 0 #endif -#if !RAPIDJSON_SCHEMA_USE_INTERNALREGEX && !defined(RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) +#if !RAPIDJSON_SCHEMA_USE_INTERNALREGEX && defined(RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) #define RAPIDJSON_SCHEMA_USE_STDREGEX 1 #else #define RAPIDJSON_SCHEMA_USE_STDREGEX 0 @@ -62,9 +63,7 @@ RAPIDJSON_DIAG_OFF(weak-vtables) RAPIDJSON_DIAG_OFF(exit-time-destructors) RAPIDJSON_DIAG_OFF(c++98-compat-pedantic) RAPIDJSON_DIAG_OFF(variadic-macros) -#endif - -#ifdef _MSC_VER +#elif defined(_MSC_VER) RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated #endif @@ -157,6 +156,62 @@ public: virtual void FreeState(void* p) = 0; }; +/////////////////////////////////////////////////////////////////////////////// +// IValidationErrorHandler + +template +class IValidationErrorHandler { +public: + typedef typename SchemaType::Ch Ch; + typedef typename SchemaType::SValue SValue; + + virtual ~IValidationErrorHandler() {} + + virtual void NotMultipleOf(int64_t actual, const SValue& expected) = 0; + virtual void NotMultipleOf(uint64_t actual, const SValue& expected) = 0; + virtual void NotMultipleOf(double actual, const SValue& expected) = 0; + virtual void AboveMaximum(int64_t actual, const SValue& expected, bool exclusive) = 0; + virtual void AboveMaximum(uint64_t actual, const SValue& expected, bool exclusive) = 0; + virtual void AboveMaximum(double actual, const SValue& expected, bool exclusive) = 0; + virtual void BelowMinimum(int64_t actual, const SValue& expected, bool exclusive) = 0; + virtual void BelowMinimum(uint64_t actual, const SValue& expected, bool exclusive) = 0; + virtual void BelowMinimum(double actual, const SValue& expected, bool exclusive) = 0; + + virtual void TooLong(const Ch* str, SizeType length, SizeType expected) = 0; + virtual void TooShort(const Ch* str, SizeType length, SizeType expected) = 0; + virtual void DoesNotMatch(const Ch* str, SizeType length) = 0; + + virtual void DisallowedItem(SizeType index) = 0; + virtual void TooFewItems(SizeType actualCount, SizeType expectedCount) = 0; + virtual void TooManyItems(SizeType actualCount, SizeType expectedCount) = 0; + virtual void DuplicateItems(SizeType index1, SizeType index2) = 0; + + virtual void TooManyProperties(SizeType actualCount, SizeType expectedCount) = 0; + virtual void TooFewProperties(SizeType actualCount, SizeType expectedCount) = 0; + virtual void StartMissingProperties() = 0; + virtual void AddMissingProperty(const SValue& name) = 0; + virtual bool EndMissingProperties() = 0; + virtual void PropertyViolations(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void DisallowedProperty(const Ch* name, SizeType length) = 0; + + virtual void StartDependencyErrors() = 0; + virtual void StartMissingDependentProperties() = 0; + virtual void AddMissingDependentProperty(const SValue& targetName) = 0; + virtual void EndMissingDependentProperties(const SValue& sourceName) = 0; + virtual void AddDependencySchemaError(const SValue& souceName, ISchemaValidator* subvalidator) = 0; + virtual bool EndDependencyErrors() = 0; + + virtual void DisallowedValue() = 0; + virtual void StartDisallowedType() = 0; + virtual void AddExpectedType(const typename SchemaType::ValueType& expectedType) = 0; + virtual void EndDisallowedType(const typename SchemaType::ValueType& actualType) = 0; + virtual void NotAllOf(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void NoneOf(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void NotOneOf(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void Disallowed() = 0; +}; + + /////////////////////////////////////////////////////////////////////////////// // Hasher @@ -261,6 +316,7 @@ template struct SchemaValidationContext { typedef Schema SchemaType; typedef ISchemaStateFactory SchemaValidatorFactoryType; + typedef IValidationErrorHandler ErrorHandlerType; typedef typename SchemaType::ValueType ValueType; typedef typename ValueType::Ch Ch; @@ -270,8 +326,9 @@ struct SchemaValidationContext { kPatternValidatorWithAdditionalProperty }; - SchemaValidationContext(SchemaValidatorFactoryType& f, const SchemaType* s) : + SchemaValidationContext(SchemaValidatorFactoryType& f, ErrorHandlerType& eh, const SchemaType* s) : factory(f), + error_handler(eh), schema(s), valueSchema(), invalidKeyword(), @@ -311,6 +368,7 @@ struct SchemaValidationContext { } SchemaValidatorFactoryType& factory; + ErrorHandlerType& error_handler; const SchemaType* schema; const SchemaType* valueSchema; const Ch* invalidKeyword; @@ -345,15 +403,20 @@ public: typedef SchemaValidationContext Context; typedef Schema SchemaType; typedef GenericValue SValue; + typedef IValidationErrorHandler ErrorHandler; friend class GenericSchemaDocument; Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : allocator_(allocator), + uri_(schemaDocument->GetURI(), *allocator), + pointer_(p, allocator), + typeless_(schemaDocument->GetTypeless()), enum_(), enumCount_(), not_(), type_((1 << kTotalSchemaType) - 1), // typeless validatorCount_(), + notValidatorIndex_(), properties_(), additionalPropertiesSchema_(), patternProperties_(), @@ -377,7 +440,8 @@ public: minLength_(0), maxLength_(~SizeType(0)), exclusiveMinimum_(false), - exclusiveMaximum_(false) + exclusiveMaximum_(false), + defaultValueLength_(0) { typedef typename SchemaDocumentType::ValueType ValueType; typedef typename ValueType::ConstValueIterator ConstValueIterator; @@ -400,7 +464,7 @@ public: enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { typedef Hasher > EnumHasherType; - char buffer[256 + 24]; + char buffer[256u + 24]; MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); EnumHasherType h(&hasherAllocator, 256); itr->Accept(h); @@ -453,7 +517,7 @@ public: for (SizeType i = 0; i < propertyCount_; i++) { new (&properties_[i]) Property(); properties_[i].name = allProperties[i]; - properties_[i].schema = GetTypeless(); + properties_[i].schema = typeless_; } } } @@ -572,12 +636,16 @@ public: if (const ValueType* v = GetMember(value, GetMultipleOfString())) if (v->IsNumber() && v->GetDouble() > 0.0) multipleOf_.CopyFrom(*v, *allocator_); + + // Default + if (const ValueType* v = GetMember(value, GetDefaultValueString())) + if (v->IsString()) + defaultValueLength_ = v->GetStringLength(); + } ~Schema() { - if (allocator_) { - allocator_->Free(enum_); - } + AllocatorType::Free(enum_); if (properties_) { for (SizeType i = 0; i < propertyCount_; i++) properties_[i].~Property(); @@ -592,11 +660,19 @@ public: #if RAPIDJSON_SCHEMA_HAS_REGEX if (pattern_) { pattern_->~RegexType(); - allocator_->Free(pattern_); + AllocatorType::Free(pattern_); } #endif } + const SValue& GetURI() const { + return uri_; + } + + const PointerType& GetPointer() const { + return pointer_; + } + bool BeginValue(Context& context) const { if (context.inArray) { if (uniqueItems_) @@ -610,12 +686,14 @@ public: else if (additionalItemsSchema_) context.valueSchema = additionalItemsSchema_; else if (additionalItems_) - context.valueSchema = GetTypeless(); - else + context.valueSchema = typeless_; + else { + context.error_handler.DisallowedItem(context.arrayElementIndex); RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); + } } else - context.valueSchema = GetTypeless(); + context.valueSchema = typeless_; context.arrayElementIndex++; } @@ -637,15 +715,21 @@ public: } if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) { - if (!patternValid) + if (!patternValid) { + context.error_handler.PropertyViolations(context.patternPropertiesValidators, count); RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } } else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) { - if (!patternValid || !otherValid) + if (!patternValid || !otherValid) { + context.error_handler.PropertyViolations(context.patternPropertiesValidators, count + 1); RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } } - else if (!patternValid && !otherValid) // kPatternValidatorWithAdditionalProperty) + else if (!patternValid && !otherValid) { // kPatternValidatorWithAdditionalProperty) + context.error_handler.PropertyViolations(context.patternPropertiesValidators, count + 1); RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } } if (enum_) { @@ -653,19 +737,23 @@ public: for (SizeType i = 0; i < enumCount_; i++) if (enum_[i] == h) goto foundEnum; + context.error_handler.DisallowedValue(); RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); foundEnum:; } if (allOf_.schemas) for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) - if (!context.validators[i]->IsValid()) + if (!context.validators[i]->IsValid()) { + context.error_handler.NotAllOf(&context.validators[allOf_.begin], allOf_.count); RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); + } if (anyOf_.schemas) { for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) if (context.validators[i]->IsValid()) goto foundAny; + context.error_handler.NoneOf(&context.validators[anyOf_.begin], anyOf_.count); RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); foundAny:; } @@ -674,30 +762,39 @@ public: bool oneValid = false; for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) if (context.validators[i]->IsValid()) { - if (oneValid) + if (oneValid) { + context.error_handler.NotOneOf(&context.validators[oneOf_.begin], oneOf_.count); RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); - else + } else oneValid = true; } - if (!oneValid) + if (!oneValid) { + context.error_handler.NotOneOf(&context.validators[oneOf_.begin], oneOf_.count); RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + } } - if (not_ && context.validators[notValidatorIndex_]->IsValid()) + if (not_ && context.validators[notValidatorIndex_]->IsValid()) { + context.error_handler.Disallowed(); RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); + } return true; } - bool Null(Context& context) const { - if (!(type_ & (1 << kNullSchemaType))) + bool Null(Context& context) const { + if (!(type_ & (1 << kNullSchemaType))) { + DisallowedType(context, GetNullString()); RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } return CreateParallelValidator(context); } - bool Bool(Context& context, bool) const { - if (!(type_ & (1 << kBooleanSchemaType))) + bool Bool(Context& context, bool) const { + if (!(type_ & (1 << kBooleanSchemaType))) { + DisallowedType(context, GetBooleanString()); RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } return CreateParallelValidator(context); } @@ -726,8 +823,10 @@ public: } bool Double(Context& context, double d) const { - if (!(type_ & (1 << kNumberSchemaType))) + if (!(type_ & (1 << kNumberSchemaType))) { + DisallowedType(context, GetNumberString()); RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) return false; @@ -742,28 +841,38 @@ public: } bool String(Context& context, const Ch* str, SizeType length, bool) const { - if (!(type_ & (1 << kStringSchemaType))) + if (!(type_ & (1 << kStringSchemaType))) { + DisallowedType(context, GetStringString()); RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } if (minLength_ != 0 || maxLength_ != SizeType(~0)) { SizeType count; if (internal::CountStringCodePoint(str, length, &count)) { - if (count < minLength_) + if (count < minLength_) { + context.error_handler.TooShort(str, length, minLength_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); - if (count > maxLength_) + } + if (count > maxLength_) { + context.error_handler.TooLong(str, length, maxLength_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); + } } } - if (pattern_ && !IsPatternMatch(pattern_, str, length)) + if (pattern_ && !IsPatternMatch(pattern_, str, length)) { + context.error_handler.DoesNotMatch(str, length); RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); + } return CreateParallelValidator(context); } - bool StartObject(Context& context) const { - if (!(type_ & (1 << kObjectSchemaType))) + bool StartObject(Context& context) const { + if (!(type_ & (1 << kObjectSchemaType))) { + DisallowedType(context, GetObjectString()); RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } if (hasDependencies_ || hasRequired_) { context.propertyExist = static_cast(context.factory.MallocState(sizeof(bool) * propertyCount_)); @@ -784,15 +893,17 @@ public: if (patternProperties_) { context.patternPropertiesSchemaCount = 0; for (SizeType i = 0; i < patternPropertyCount_; i++) - if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) + if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) { context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; + context.valueSchema = typeless_; + } } SizeType index; if (FindPropertyIndex(ValueType(str, len).Move(), &index)) { if (context.patternPropertiesSchemaCount > 0) { context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; - context.valueSchema = GetTypeless(); + context.valueSchema = typeless_; context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; } else @@ -807,7 +918,7 @@ public: if (additionalPropertiesSchema_) { if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; - context.valueSchema = GetTypeless(); + context.valueSchema = typeless_; context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; } else @@ -815,49 +926,70 @@ public: return true; } else if (additionalProperties_) { - context.valueSchema = GetTypeless(); + context.valueSchema = typeless_; return true; } - if (context.patternPropertiesSchemaCount == 0) // patternProperties are not additional properties + if (context.patternPropertiesSchemaCount == 0) { // patternProperties are not additional properties + context.error_handler.DisallowedProperty(str, len); RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); + } return true; } bool EndObject(Context& context, SizeType memberCount) const { - if (hasRequired_) + if (hasRequired_) { + context.error_handler.StartMissingProperties(); for (SizeType index = 0; index < propertyCount_; index++) - if (properties_[index].required) - if (!context.propertyExist[index]) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); + if (properties_[index].required && !context.propertyExist[index]) + if (properties_[index].schema->defaultValueLength_ == 0 ) + context.error_handler.AddMissingProperty(properties_[index].name); + if (context.error_handler.EndMissingProperties()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); + } - if (memberCount < minProperties_) + if (memberCount < minProperties_) { + context.error_handler.TooFewProperties(memberCount, minProperties_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); + } - if (memberCount > maxProperties_) + if (memberCount > maxProperties_) { + context.error_handler.TooManyProperties(memberCount, maxProperties_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); + } if (hasDependencies_) { - for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) + context.error_handler.StartDependencyErrors(); + for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) { + const Property& source = properties_[sourceIndex]; if (context.propertyExist[sourceIndex]) { - if (properties_[sourceIndex].dependencies) { + if (source.dependencies) { + context.error_handler.StartMissingDependentProperties(); for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) - if (properties_[sourceIndex].dependencies[targetIndex] && !context.propertyExist[targetIndex]) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + if (source.dependencies[targetIndex] && !context.propertyExist[targetIndex]) + context.error_handler.AddMissingDependentProperty(properties_[targetIndex].name); + context.error_handler.EndMissingDependentProperties(source.name); + } + else if (source.dependenciesSchema) { + ISchemaValidator* dependenciesValidator = context.validators[source.dependenciesValidatorIndex]; + if (!dependenciesValidator->IsValid()) + context.error_handler.AddDependencySchemaError(source.name, dependenciesValidator); } - else if (properties_[sourceIndex].dependenciesSchema) - if (!context.validators[properties_[sourceIndex].dependenciesValidatorIndex]->IsValid()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); } + } + if (context.error_handler.EndDependencyErrors()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); } return true; } - bool StartArray(Context& context) const { - if (!(type_ & (1 << kArraySchemaType))) + bool StartArray(Context& context) const { + if (!(type_ & (1 << kArraySchemaType))) { + DisallowedType(context, GetArrayString()); RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } context.arrayElementIndex = 0; context.inArray = true; @@ -865,14 +997,18 @@ public: return CreateParallelValidator(context); } - bool EndArray(Context& context, SizeType elementCount) const { + bool EndArray(Context& context, SizeType elementCount) const { context.inArray = false; - if (elementCount < minItems_) + if (elementCount < minItems_) { + context.error_handler.TooFewItems(elementCount, minItems_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); + } - if (elementCount > maxItems_) + if (elementCount > maxItems_) { + context.error_handler.TooManyItems(elementCount, maxItems_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); + } return true; } @@ -881,7 +1017,7 @@ public: #define RAPIDJSON_STRING_(name, ...) \ static const ValueType& Get##name##String() {\ static const Ch s[] = { __VA_ARGS__, '\0' };\ - static const ValueType v(s, sizeof(s) / sizeof(Ch) - 1);\ + static const ValueType v(s, static_cast(sizeof(s) / sizeof(Ch) - 1));\ return v;\ } @@ -918,6 +1054,7 @@ public: RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') + RAPIDJSON_STRING_(DefaultValue, 'd', 'e', 'f', 'a', 'u', 'l', 't') #undef RAPIDJSON_STRING_ @@ -934,7 +1071,7 @@ private: }; #if RAPIDJSON_SCHEMA_USE_INTERNALREGEX - typedef internal::GenericRegex RegexType; + typedef internal::GenericRegex RegexType; #elif RAPIDJSON_SCHEMA_USE_STDREGEX typedef std::basic_regex RegexType; #else @@ -949,11 +1086,6 @@ private: SizeType count; }; - static const SchemaType* GetTypeless() { - static SchemaType typeless(0, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), 0); - return &typeless; - } - template void AddUniqueElement(V1& a, const V2& v) { for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) @@ -999,7 +1131,7 @@ private: template RegexType* CreatePattern(const ValueType& value) { if (value.IsString()) { - RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString()); + RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), allocator_); if (!r->IsValid()) { r->~RegexType(); AllocatorType::Free(r); @@ -1011,17 +1143,21 @@ private: } static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) { - return pattern->Search(str); + GenericRegexSearch rs(*pattern); + return rs.Search(str); } #elif RAPIDJSON_SCHEMA_USE_STDREGEX template RegexType* CreatePattern(const ValueType& value) { - if (value.IsString()) + if (value.IsString()) { + RegexType *r = static_cast(allocator_->Malloc(sizeof(RegexType))); try { - return new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); + return new (r) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); } catch (const std::regex_error&) { + AllocatorType::Free(r); } + } return 0; } @@ -1097,15 +1233,20 @@ private: } bool CheckInt(Context& context, int64_t i) const { - if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) { + DisallowedType(context, GetIntegerString()); RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } if (!minimum_.IsNull()) { if (minimum_.IsInt64()) { - if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) + if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) { + context.error_handler.BelowMinimum(i, minimum_, exclusiveMinimum_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } } else if (minimum_.IsUint64()) { + context.error_handler.BelowMinimum(i, minimum_, exclusiveMinimum_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() } else if (!CheckDoubleMinimum(context, static_cast(i))) @@ -1114,19 +1255,23 @@ private: if (!maximum_.IsNull()) { if (maximum_.IsInt64()) { - if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) + if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) { + context.error_handler.AboveMaximum(i, maximum_, exclusiveMaximum_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } } - else if (maximum_.IsUint64()) - /* do nothing */; // i <= max(int64_t) < maximum_.GetUint64() + else if (maximum_.IsUint64()) { } + /* do nothing */ // i <= max(int64_t) < maximum_.GetUint64() else if (!CheckDoubleMaximum(context, static_cast(i))) return false; } if (!multipleOf_.IsNull()) { if (multipleOf_.IsUint64()) { - if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) + if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) { + context.error_handler.NotMultipleOf(i, multipleOf_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } } else if (!CheckDoubleMultipleOf(context, static_cast(i))) return false; @@ -1136,13 +1281,17 @@ private: } bool CheckUint(Context& context, uint64_t i) const { - if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) { + DisallowedType(context, GetIntegerString()); RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } if (!minimum_.IsNull()) { if (minimum_.IsUint64()) { - if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) + if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) { + context.error_handler.BelowMinimum(i, minimum_, exclusiveMinimum_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } } else if (minimum_.IsInt64()) /* do nothing */; // i >= 0 > minimum.Getint64() @@ -1152,19 +1301,25 @@ private: if (!maximum_.IsNull()) { if (maximum_.IsUint64()) { - if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) + if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) { + context.error_handler.AboveMaximum(i, maximum_, exclusiveMaximum_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } } - else if (maximum_.IsInt64()) + else if (maximum_.IsInt64()) { + context.error_handler.AboveMaximum(i, maximum_, exclusiveMaximum_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ + } else if (!CheckDoubleMaximum(context, static_cast(i))) return false; } if (!multipleOf_.IsNull()) { if (multipleOf_.IsUint64()) { - if (i % multipleOf_.GetUint64() != 0) + if (i % multipleOf_.GetUint64() != 0) { + context.error_handler.NotMultipleOf(i, multipleOf_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } } else if (!CheckDoubleMultipleOf(context, static_cast(i))) return false; @@ -1174,14 +1329,18 @@ private: } bool CheckDoubleMinimum(Context& context, double d) const { - if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) + if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) { + context.error_handler.BelowMinimum(d, minimum_, exclusiveMinimum_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } return true; } bool CheckDoubleMaximum(Context& context, double d) const { - if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) + if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) { + context.error_handler.AboveMaximum(d, maximum_, exclusiveMaximum_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } return true; } @@ -1189,11 +1348,29 @@ private: double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); double q = std::floor(a / b); double r = a - q * b; - if (r > 0.0) + if (r > 0.0) { + context.error_handler.NotMultipleOf(d, multipleOf_); RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } return true; } + void DisallowedType(Context& context, const ValueType& actualType) const { + ErrorHandler& eh = context.error_handler; + eh.StartDisallowedType(); + + if (type_ & (1 << kNullSchemaType)) eh.AddExpectedType(GetNullString()); + if (type_ & (1 << kBooleanSchemaType)) eh.AddExpectedType(GetBooleanString()); + if (type_ & (1 << kObjectSchemaType)) eh.AddExpectedType(GetObjectString()); + if (type_ & (1 << kArraySchemaType)) eh.AddExpectedType(GetArrayString()); + if (type_ & (1 << kStringSchemaType)) eh.AddExpectedType(GetStringString()); + + if (type_ & (1 << kNumberSchemaType)) eh.AddExpectedType(GetNumberString()); + else if (type_ & (1 << kIntegerSchemaType)) eh.AddExpectedType(GetIntegerString()); + + eh.EndDisallowedType(actualType); + } + struct Property { Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} ~Property() { AllocatorType::Free(dependencies); } @@ -1218,6 +1395,9 @@ private: }; AllocatorType* allocator_; + SValue uri_; + PointerType pointer_; + const SchemaType* typeless_; uint64_t* enum_; SizeType enumCount_; SchemaArray allOf_; @@ -1258,6 +1438,8 @@ private: SValue multipleOf_; bool exclusiveMinimum_; bool exclusiveMaximum_; + + SizeType defaultValueLength_; }; template @@ -1267,7 +1449,7 @@ struct TokenHelper { char buffer[21]; size_t length = static_cast((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); for (size_t i = 0; i < length; i++) - *documentStack.template Push() = buffer[i]; + *documentStack.template Push() = static_cast(buffer[i]); } }; @@ -1326,6 +1508,7 @@ public: typedef typename EncodingType::Ch Ch; typedef internal::Schema SchemaType; typedef GenericPointer PointerType; + typedef GenericValue URIType; friend class internal::Schema; template friend class GenericSchemaValidator; @@ -1335,19 +1518,29 @@ public: Compile a JSON document into schema document. \param document A JSON document as source. + \param uri The base URI of this schema document for purposes of violation reporting. + \param uriLength Length of \c name, in code points. \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. \param allocator An optional allocator instance for allocating memory. Can be null. */ - explicit GenericSchemaDocument(const ValueType& document, IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : + explicit GenericSchemaDocument(const ValueType& document, const Ch* uri = 0, SizeType uriLength = 0, + IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : remoteProvider_(remoteProvider), allocator_(allocator), ownAllocator_(), root_(), + typeless_(), schemaMap_(allocator, kInitialSchemaMapSize), schemaRef_(allocator, kInitialSchemaRefSize) { if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator()); + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + + Ch noUri[1] = {0}; + uri_.SetString(uri ? uri : noUri, uriLength, *allocator_); + + typeless_ = static_cast(allocator_->Malloc(sizeof(SchemaType))); + new (typeless_) SchemaType(this, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), allocator_); // Generate root schema, it will call CreateSchema() to create sub-schemas, // And call AddRefSchema() if there are $ref. @@ -1365,6 +1558,9 @@ public: new (schemaMap_.template Push()) SchemaEntry(refEntry->source, const_cast(s), false, allocator_); } } + else if (refEntry->schema) + *refEntry->schema = typeless_; + refEntry->~SchemaRefEntry(); } @@ -1380,12 +1576,15 @@ public: allocator_(rhs.allocator_), ownAllocator_(rhs.ownAllocator_), root_(rhs.root_), + typeless_(rhs.typeless_), schemaMap_(std::move(rhs.schemaMap_)), - schemaRef_(std::move(rhs.schemaRef_)) + schemaRef_(std::move(rhs.schemaRef_)), + uri_(std::move(rhs.uri_)) { rhs.remoteProvider_ = 0; rhs.allocator_ = 0; rhs.ownAllocator_ = 0; + rhs.typeless_ = 0; } #endif @@ -1394,9 +1593,16 @@ public: while (!schemaMap_.Empty()) schemaMap_.template Pop(1)->~SchemaEntry(); + if (typeless_) { + typeless_->~SchemaType(); + Allocator::Free(typeless_); + } + RAPIDJSON_DELETE(ownAllocator_); } + const URIType& GetURI() const { return uri_; } + //! Get the root schema. const SchemaType& GetRoot() const { return *root_; } @@ -1428,7 +1634,7 @@ private: void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { if (schema) - *schema = SchemaType::GetTypeless(); + *schema = typeless_; if (v.GetType() == kObjectType) { const SchemaType* s = GetSchema(pointer); @@ -1473,12 +1679,13 @@ private: if (i > 0) { // Remote reference, resolve immediately if (remoteProvider_) { - if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i - 1)) { + if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i)) { PointerType pointer(&s[i], len - i, allocator_); if (pointer.IsValid()) { if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) { if (schema) *schema = sc; + new (schemaMap_.template Push()) SchemaEntry(source, const_cast(sc), false, allocator_); return true; } } @@ -1515,6 +1722,8 @@ private: return PointerType(); } + const SchemaType* GetTypeless() const { return typeless_; } + static const size_t kInitialSchemaMapSize = 64; static const size_t kInitialSchemaRefSize = 64; @@ -1522,8 +1731,10 @@ private: Allocator *allocator_; Allocator *ownAllocator_; const SchemaType* root_; //!< Root schema. + SchemaType* typeless_; internal::Stack schemaMap_; // Stores created Pointer -> Schemas internal::Stack schemaRef_; // Stores Pointer from $ref and schema which holds the $ref + URIType uri_; }; //! GenericSchemaDocument using Value type. @@ -1552,13 +1763,17 @@ template < typename StateAllocator = CrtAllocator> class GenericSchemaValidator : public internal::ISchemaStateFactory, - public internal::ISchemaValidator + public internal::ISchemaValidator, + public internal::IValidationErrorHandler { public: typedef typename SchemaDocumentType::SchemaType SchemaType; typedef typename SchemaDocumentType::PointerType PointerType; typedef typename SchemaType::EncodingType EncodingType; + typedef typename SchemaType::SValue SValue; typedef typename EncodingType::Ch Ch; + typedef GenericStringRef StringRefType; + typedef GenericValue ValueType; //! Constructor without output handler. /*! @@ -1575,11 +1790,14 @@ public: : schemaDocument_(&schemaDocument), root_(schemaDocument.GetRoot()), - outputHandler_(GetNullHandler()), stateAllocator_(allocator), ownStateAllocator_(0), schemaStack_(allocator, schemaStackCapacity), documentStack_(allocator, documentStackCapacity), + outputHandler_(0), + error_(kObjectType), + currentError_(), + missingDependents_(), valid_(true) #if RAPIDJSON_SCHEMA_VERBOSE , depth_(0) @@ -1603,11 +1821,14 @@ public: : schemaDocument_(&schemaDocument), root_(schemaDocument.GetRoot()), - outputHandler_(outputHandler), stateAllocator_(allocator), ownStateAllocator_(0), schemaStack_(allocator, schemaStackCapacity), documentStack_(allocator, documentStackCapacity), + outputHandler_(&outputHandler), + error_(kObjectType), + currentError_(), + missingDependents_(), valid_(true) #if RAPIDJSON_SCHEMA_VERBOSE , depth_(0) @@ -1626,6 +1847,9 @@ public: while (!schemaStack_.Empty()) PopSchema(); documentStack_.Clear(); + error_.SetObject(); + currentError_.SetNull(); + missingDependents_.SetNull(); valid_ = true; } @@ -1633,9 +1857,13 @@ public: // Implementation of ISchemaValidator virtual bool IsValid() const { return valid_; } + //! Gets the error object. + ValueType& GetError() { return error_; } + const ValueType& GetError() const { return error_; } + //! Gets the JSON pointer pointed to the invalid schema. PointerType GetInvalidSchemaPointer() const { - return schemaStack_.Empty() ? PointerType() : schemaDocument_->GetPointer(&CurrentSchema()); + return schemaStack_.Empty() ? PointerType() : CurrentSchema().GetPointer(); } //! Gets the keyword of invalid schema. @@ -1645,9 +1873,196 @@ public: //! Gets the JSON pointer pointed to the invalid value. PointerType GetInvalidDocumentPointer() const { - return documentStack_.Empty() ? PointerType() : PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); + if (documentStack_.Empty()) { + return PointerType(); + } + else { + return PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); + } } + void NotMultipleOf(int64_t actual, const SValue& expected) { + AddNumberError(SchemaType::GetMultipleOfString(), ValueType(actual).Move(), expected); + } + void NotMultipleOf(uint64_t actual, const SValue& expected) { + AddNumberError(SchemaType::GetMultipleOfString(), ValueType(actual).Move(), expected); + } + void NotMultipleOf(double actual, const SValue& expected) { + AddNumberError(SchemaType::GetMultipleOfString(), ValueType(actual).Move(), expected); + } + void AboveMaximum(int64_t actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMaximumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMaximumString : 0); + } + void AboveMaximum(uint64_t actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMaximumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMaximumString : 0); + } + void AboveMaximum(double actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMaximumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMaximumString : 0); + } + void BelowMinimum(int64_t actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMinimumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMinimumString : 0); + } + void BelowMinimum(uint64_t actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMinimumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMinimumString : 0); + } + void BelowMinimum(double actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMinimumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMinimumString : 0); + } + + void TooLong(const Ch* str, SizeType length, SizeType expected) { + AddNumberError(SchemaType::GetMaxLengthString(), + ValueType(str, length, GetStateAllocator()).Move(), SValue(expected).Move()); + } + void TooShort(const Ch* str, SizeType length, SizeType expected) { + AddNumberError(SchemaType::GetMinLengthString(), + ValueType(str, length, GetStateAllocator()).Move(), SValue(expected).Move()); + } + void DoesNotMatch(const Ch* str, SizeType length) { + currentError_.SetObject(); + currentError_.AddMember(GetActualString(), ValueType(str, length, GetStateAllocator()).Move(), GetStateAllocator()); + AddCurrentError(SchemaType::GetPatternString()); + } + + void DisallowedItem(SizeType index) { + currentError_.SetObject(); + currentError_.AddMember(GetDisallowedString(), ValueType(index).Move(), GetStateAllocator()); + AddCurrentError(SchemaType::GetAdditionalItemsString(), true); + } + void TooFewItems(SizeType actualCount, SizeType expectedCount) { + AddNumberError(SchemaType::GetMinItemsString(), + ValueType(actualCount).Move(), SValue(expectedCount).Move()); + } + void TooManyItems(SizeType actualCount, SizeType expectedCount) { + AddNumberError(SchemaType::GetMaxItemsString(), + ValueType(actualCount).Move(), SValue(expectedCount).Move()); + } + void DuplicateItems(SizeType index1, SizeType index2) { + ValueType duplicates(kArrayType); + duplicates.PushBack(index1, GetStateAllocator()); + duplicates.PushBack(index2, GetStateAllocator()); + currentError_.SetObject(); + currentError_.AddMember(GetDuplicatesString(), duplicates, GetStateAllocator()); + AddCurrentError(SchemaType::GetUniqueItemsString(), true); + } + + void TooManyProperties(SizeType actualCount, SizeType expectedCount) { + AddNumberError(SchemaType::GetMaxPropertiesString(), + ValueType(actualCount).Move(), SValue(expectedCount).Move()); + } + void TooFewProperties(SizeType actualCount, SizeType expectedCount) { + AddNumberError(SchemaType::GetMinPropertiesString(), + ValueType(actualCount).Move(), SValue(expectedCount).Move()); + } + void StartMissingProperties() { + currentError_.SetArray(); + } + void AddMissingProperty(const SValue& name) { + currentError_.PushBack(ValueType(name, GetStateAllocator()).Move(), GetStateAllocator()); + } + bool EndMissingProperties() { + if (currentError_.Empty()) + return false; + ValueType error(kObjectType); + error.AddMember(GetMissingString(), currentError_, GetStateAllocator()); + currentError_ = error; + AddCurrentError(SchemaType::GetRequiredString()); + return true; + } + void PropertyViolations(ISchemaValidator** subvalidators, SizeType count) { + for (SizeType i = 0; i < count; ++i) + MergeError(static_cast(subvalidators[i])->GetError()); + } + void DisallowedProperty(const Ch* name, SizeType length) { + currentError_.SetObject(); + currentError_.AddMember(GetDisallowedString(), ValueType(name, length, GetStateAllocator()).Move(), GetStateAllocator()); + AddCurrentError(SchemaType::GetAdditionalPropertiesString(), true); + } + + void StartDependencyErrors() { + currentError_.SetObject(); + } + void StartMissingDependentProperties() { + missingDependents_.SetArray(); + } + void AddMissingDependentProperty(const SValue& targetName) { + missingDependents_.PushBack(ValueType(targetName, GetStateAllocator()).Move(), GetStateAllocator()); + } + void EndMissingDependentProperties(const SValue& sourceName) { + if (!missingDependents_.Empty()) + currentError_.AddMember(ValueType(sourceName, GetStateAllocator()).Move(), + missingDependents_, GetStateAllocator()); + } + void AddDependencySchemaError(const SValue& sourceName, ISchemaValidator* subvalidator) { + currentError_.AddMember(ValueType(sourceName, GetStateAllocator()).Move(), + static_cast(subvalidator)->GetError(), GetStateAllocator()); + } + bool EndDependencyErrors() { + if (currentError_.ObjectEmpty()) + return false; + ValueType error(kObjectType); + error.AddMember(GetErrorsString(), currentError_, GetStateAllocator()); + currentError_ = error; + AddCurrentError(SchemaType::GetDependenciesString()); + return true; + } + + void DisallowedValue() { + currentError_.SetObject(); + AddCurrentError(SchemaType::GetEnumString()); + } + void StartDisallowedType() { + currentError_.SetArray(); + } + void AddExpectedType(const typename SchemaType::ValueType& expectedType) { + currentError_.PushBack(ValueType(expectedType, GetStateAllocator()).Move(), GetStateAllocator()); + } + void EndDisallowedType(const typename SchemaType::ValueType& actualType) { + ValueType error(kObjectType); + error.AddMember(GetExpectedString(), currentError_, GetStateAllocator()); + error.AddMember(GetActualString(), ValueType(actualType, GetStateAllocator()).Move(), GetStateAllocator()); + currentError_ = error; + AddCurrentError(SchemaType::GetTypeString()); + } + void NotAllOf(ISchemaValidator** subvalidators, SizeType count) { + for (SizeType i = 0; i < count; ++i) { + MergeError(static_cast(subvalidators[i])->GetError()); + } + } + void NoneOf(ISchemaValidator** subvalidators, SizeType count) { + AddErrorArray(SchemaType::GetAnyOfString(), subvalidators, count); + } + void NotOneOf(ISchemaValidator** subvalidators, SizeType count) { + AddErrorArray(SchemaType::GetOneOfString(), subvalidators, count); + } + void Disallowed() { + currentError_.SetObject(); + AddCurrentError(SchemaType::GetNotString()); + } + +#define RAPIDJSON_STRING_(name, ...) \ + static const StringRefType& Get##name##String() {\ + static const Ch s[] = { __VA_ARGS__, '\0' };\ + static const StringRefType v(s, static_cast(sizeof(s) / sizeof(Ch) - 1)); \ + return v;\ + } + + RAPIDJSON_STRING_(InstanceRef, 'i', 'n', 's', 't', 'a', 'n', 'c', 'e', 'R', 'e', 'f') + RAPIDJSON_STRING_(SchemaRef, 's', 'c', 'h', 'e', 'm', 'a', 'R', 'e', 'f') + RAPIDJSON_STRING_(Expected, 'e', 'x', 'p', 'e', 'c', 't', 'e', 'd') + RAPIDJSON_STRING_(Actual, 'a', 'c', 't', 'u', 'a', 'l') + RAPIDJSON_STRING_(Disallowed, 'd', 'i', 's', 'a', 'l', 'l', 'o', 'w', 'e', 'd') + RAPIDJSON_STRING_(Missing, 'm', 'i', 's', 's', 'i', 'n', 'g') + RAPIDJSON_STRING_(Errors, 'e', 'r', 'r', 'o', 'r', 's') + RAPIDJSON_STRING_(Duplicates, 'd', 'u', 'p', 'l', 'i', 'c', 'a', 't', 'e', 's') + +#undef RAPIDJSON_STRING_ + #if RAPIDJSON_SCHEMA_VERBOSE #define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ RAPIDJSON_MULTILINEMACRO_BEGIN\ @@ -1679,14 +2094,14 @@ RAPIDJSON_MULTILINEMACRO_END } #define RAPIDJSON_SCHEMA_HANDLE_END_(method, arg2)\ - return valid_ = EndValue() && outputHandler_.method arg2 + return valid_ = EndValue() && (!outputHandler_ || outputHandler_->method arg2) #define RAPIDJSON_SCHEMA_HANDLE_VALUE_(method, arg1, arg2) \ RAPIDJSON_SCHEMA_HANDLE_BEGIN_ (method, arg1);\ RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2);\ RAPIDJSON_SCHEMA_HANDLE_END_ (method, arg2) - bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext() ), ( )); } + bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext()), ( )); } bool Bool(bool b) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } bool Int(int i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } bool Uint(unsigned u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } @@ -1701,7 +2116,7 @@ RAPIDJSON_MULTILINEMACRO_END bool StartObject() { RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); - return valid_ = outputHandler_.StartObject(); + return valid_ = !outputHandler_ || outputHandler_->StartObject(); } bool Key(const Ch* str, SizeType len, bool copy) { @@ -1709,7 +2124,7 @@ RAPIDJSON_MULTILINEMACRO_END AppendToken(str, len); if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); - return valid_ = outputHandler_.Key(str, len, copy); + return valid_ = !outputHandler_ || outputHandler_->Key(str, len, copy); } bool EndObject(SizeType memberCount) { @@ -1722,7 +2137,7 @@ RAPIDJSON_MULTILINEMACRO_END bool StartArray() { RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); - return valid_ = outputHandler_.StartArray(); + return valid_ = !outputHandler_ || outputHandler_->StartArray(); } bool EndArray(SizeType elementCount) { @@ -1739,7 +2154,7 @@ RAPIDJSON_MULTILINEMACRO_END // Implementation of ISchemaStateFactory virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) { - return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, + return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, documentStack_.template Bottom(), documentStack_.GetSize(), #if RAPIDJSON_SCHEMA_VERBOSE depth_ + 1, #endif @@ -1771,7 +2186,7 @@ RAPIDJSON_MULTILINEMACRO_END } virtual void FreeState(void* p) { - return StateAllocator::Free(p); + StateAllocator::Free(p); } private: @@ -1782,6 +2197,7 @@ private: GenericSchemaValidator( const SchemaDocumentType& schemaDocument, const SchemaType& root, + const char* basePath, size_t basePathSize, #if RAPIDJSON_SCHEMA_VERBOSE unsigned depth, #endif @@ -1791,21 +2207,26 @@ private: : schemaDocument_(&schemaDocument), root_(root), - outputHandler_(GetNullHandler()), stateAllocator_(allocator), ownStateAllocator_(0), schemaStack_(allocator, schemaStackCapacity), documentStack_(allocator, documentStackCapacity), + outputHandler_(0), + error_(kObjectType), + currentError_(), + missingDependents_(), valid_(true) #if RAPIDJSON_SCHEMA_VERBOSE , depth_(depth) #endif { + if (basePath && basePathSize) + memcpy(documentStack_.template Push(basePathSize), basePath, basePathSize); } StateAllocator& GetStateAllocator() { if (!stateAllocator_) - stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator()); + stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator)(); return *stateAllocator_; } @@ -1823,8 +2244,8 @@ private: const SchemaType** sa = CurrentContext().patternPropertiesSchemas; typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; bool valueUniqueness = CurrentContext().valueUniqueness; - if (CurrentContext().valueSchema) - PushSchema(*CurrentContext().valueSchema); + RAPIDJSON_ASSERT(CurrentContext().valueSchema); + PushSchema(*CurrentContext().valueSchema); if (count > 0) { CurrentContext().objectPatternValidatorType = patternValidatorType; @@ -1864,8 +2285,10 @@ private: if (!a) CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) - if (itr->GetUint64() == h) + if (itr->GetUint64() == h) { + DuplicateItems(static_cast(itr - a->Begin()), a->Size()); RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); + } a->PushBack(h, GetStateAllocator()); } } @@ -1894,7 +2317,7 @@ private: } } - RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, &schema); } + RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, *this, &schema); } RAPIDJSON_FORCEINLINE void PopSchema() { Context* c = schemaStack_.template Pop(1); @@ -1905,24 +2328,86 @@ private: c->~Context(); } + void AddErrorLocation(ValueType& result, bool parent) { + GenericStringBuffer sb; + PointerType instancePointer = GetInvalidDocumentPointer(); + ((parent && instancePointer.GetTokenCount() > 0) + ? PointerType(instancePointer.GetTokens(), instancePointer.GetTokenCount() - 1) + : instancePointer).StringifyUriFragment(sb); + ValueType instanceRef(sb.GetString(), static_cast(sb.GetSize() / sizeof(Ch)), + GetStateAllocator()); + result.AddMember(GetInstanceRefString(), instanceRef, GetStateAllocator()); + sb.Clear(); + memcpy(sb.Push(CurrentSchema().GetURI().GetStringLength()), + CurrentSchema().GetURI().GetString(), + CurrentSchema().GetURI().GetStringLength() * sizeof(Ch)); + GetInvalidSchemaPointer().StringifyUriFragment(sb); + ValueType schemaRef(sb.GetString(), static_cast(sb.GetSize() / sizeof(Ch)), + GetStateAllocator()); + result.AddMember(GetSchemaRefString(), schemaRef, GetStateAllocator()); + } + + void AddError(ValueType& keyword, ValueType& error) { + typename ValueType::MemberIterator member = error_.FindMember(keyword); + if (member == error_.MemberEnd()) + error_.AddMember(keyword, error, GetStateAllocator()); + else { + if (member->value.IsObject()) { + ValueType errors(kArrayType); + errors.PushBack(member->value, GetStateAllocator()); + member->value = errors; + } + member->value.PushBack(error, GetStateAllocator()); + } + } + + void AddCurrentError(const typename SchemaType::ValueType& keyword, bool parent = false) { + AddErrorLocation(currentError_, parent); + AddError(ValueType(keyword, GetStateAllocator(), false).Move(), currentError_); + } + + void MergeError(ValueType& other) { + for (typename ValueType::MemberIterator it = other.MemberBegin(), end = other.MemberEnd(); it != end; ++it) { + AddError(it->name, it->value); + } + } + + void AddNumberError(const typename SchemaType::ValueType& keyword, ValueType& actual, const SValue& expected, + const typename SchemaType::ValueType& (*exclusive)() = 0) { + currentError_.SetObject(); + currentError_.AddMember(GetActualString(), actual, GetStateAllocator()); + currentError_.AddMember(GetExpectedString(), ValueType(expected, GetStateAllocator()).Move(), GetStateAllocator()); + if (exclusive) + currentError_.AddMember(ValueType(exclusive(), GetStateAllocator()).Move(), true, GetStateAllocator()); + AddCurrentError(keyword); + } + + void AddErrorArray(const typename SchemaType::ValueType& keyword, + ISchemaValidator** subvalidators, SizeType count) { + ValueType errors(kArrayType); + for (SizeType i = 0; i < count; ++i) + errors.PushBack(static_cast(subvalidators[i])->GetError(), GetStateAllocator()); + currentError_.SetObject(); + currentError_.AddMember(GetErrorsString(), errors, GetStateAllocator()); + AddCurrentError(keyword); + } + const SchemaType& CurrentSchema() const { return *schemaStack_.template Top()->schema; } Context& CurrentContext() { return *schemaStack_.template Top(); } const Context& CurrentContext() const { return *schemaStack_.template Top(); } - static OutputHandler& GetNullHandler() { - static OutputHandler nullHandler; - return nullHandler; - } - static const size_t kDefaultSchemaStackCapacity = 1024; static const size_t kDefaultDocumentStackCapacity = 256; const SchemaDocumentType* schemaDocument_; const SchemaType& root_; - OutputHandler& outputHandler_; StateAllocator* stateAllocator_; StateAllocator* ownStateAllocator_; internal::Stack schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) internal::Stack documentStack_; //!< stack to store the current path of validating document (Ch) + OutputHandler* outputHandler_; + ValueType error_; + ValueType currentError_; + ValueType missingDependents_; bool valid_; #if RAPIDJSON_SCHEMA_VERBOSE unsigned depth_; @@ -1954,13 +2439,14 @@ class SchemaValidatingReader { public: typedef typename SchemaDocumentType::PointerType PointerType; typedef typename InputStream::Ch Ch; + typedef GenericValue ValueType; //! Constructor /*! \param is Input stream. \param sd Schema document. */ - SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), isValid_(true) {} + SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), error_(kObjectType), isValid_(true) {} template bool operator()(Handler& handler) { @@ -1973,11 +2459,13 @@ public: invalidSchemaPointer_ = PointerType(); invalidSchemaKeyword_ = 0; invalidDocumentPointer_ = PointerType(); + error_.SetObject(); } else { invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); + error_.CopyFrom(validator.GetError(), allocator_); } return parseResult_; @@ -1988,6 +2476,7 @@ public: const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } + const ValueType& GetError() const { return error_; } private: InputStream& is_; @@ -1997,6 +2486,8 @@ private: PointerType invalidSchemaPointer_; const Ch* invalidSchemaKeyword_; PointerType invalidDocumentPointer_; + StackAllocator allocator_; + ValueType error_; bool isValid_; }; diff --git a/src/3rdparty/rapidjson/stream.h b/src/3rdparty/rapidjson/stream.h index fef82c25..7f2643e4 100644 --- a/src/3rdparty/rapidjson/stream.h +++ b/src/3rdparty/rapidjson/stream.h @@ -1,5 +1,5 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// +// // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except @@ -7,9 +7,9 @@ // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #include "rapidjson.h" @@ -100,6 +100,50 @@ inline void PutN(Stream& stream, Ch c, size_t n) { PutUnsafe(stream, c); } +/////////////////////////////////////////////////////////////////////////////// +// GenericStreamWrapper + +//! A Stream Wrapper +/*! \tThis string stream is a wrapper for any stream by just forwarding any + \treceived message to the origin stream. + \note implements Stream concept +*/ + +#if defined(_MSC_VER) && _MSC_VER <= 1800 +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4702) // unreachable code +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +template > +class GenericStreamWrapper { +public: + typedef typename Encoding::Ch Ch; + GenericStreamWrapper(InputStream& is): is_(is) {} + + Ch Peek() const { return is_.Peek(); } + Ch Take() { return is_.Take(); } + size_t Tell() { return is_.Tell(); } + Ch* PutBegin() { return is_.PutBegin(); } + void Put(Ch ch) { is_.Put(ch); } + void Flush() { is_.Flush(); } + size_t PutEnd(Ch* ch) { return is_.PutEnd(ch); } + + // wrapper for MemoryStream + const Ch* Peek4() const { return is_.Peek4(); } + + // wrapper for AutoUTFInputStream + UTFType GetType() const { return is_.GetType(); } + bool HasBOM() const { return is_.HasBOM(); } + +protected: + InputStream& is_; +}; + +#if defined(_MSC_VER) && _MSC_VER <= 1800 +RAPIDJSON_DIAG_POP +#endif + /////////////////////////////////////////////////////////////////////////////// // StringStream diff --git a/src/3rdparty/rapidjson/stringbuffer.h b/src/3rdparty/rapidjson/stringbuffer.h index 78f34d20..4e38b82c 100644 --- a/src/3rdparty/rapidjson/stringbuffer.h +++ b/src/3rdparty/rapidjson/stringbuffer.h @@ -78,8 +78,12 @@ public: return stack_.template Bottom(); } + //! Get the size of string in bytes in the string buffer. size_t GetSize() const { return stack_.GetSize(); } + //! Get the length of string in Ch in the string buffer. + size_t GetLength() const { return stack_.GetSize() / sizeof(Ch); } + static const size_t kDefaultCapacity = 256; mutable internal::Stack stack_; diff --git a/src/3rdparty/rapidjson/writer.h b/src/3rdparty/rapidjson/writer.h index 94f22dd5..6f5b6903 100644 --- a/src/3rdparty/rapidjson/writer.h +++ b/src/3rdparty/rapidjson/writer.h @@ -16,6 +16,7 @@ #define RAPIDJSON_WRITER_H_ #include "stream.h" +#include "internal/meta.h" #include "internal/stack.h" #include "internal/strfunc.h" #include "internal/dtoa.h" @@ -31,17 +32,18 @@ #include #elif defined(RAPIDJSON_SSE2) #include -#endif - -#ifdef _MSC_VER -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant +#elif defined(RAPIDJSON_NEON) +#include #endif #ifdef __clang__ RAPIDJSON_DIAG_PUSH RAPIDJSON_DIAG_OFF(padded) RAPIDJSON_DIAG_OFF(unreachable-code) +RAPIDJSON_DIAG_OFF(c++98-compat) +#elif defined(_MSC_VER) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant #endif RAPIDJSON_NAMESPACE_BEGIN @@ -103,6 +105,13 @@ public: Writer(StackAllocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : os_(0), level_stack_(allocator, levelDepth * sizeof(Level)), maxDecimalPlaces_(kDefaultMaxDecimalPlaces), hasRoot_(false) {} +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + Writer(Writer&& rhs) : + os_(rhs.os_), level_stack_(std::move(rhs.level_stack_)), maxDecimalPlaces_(rhs.maxDecimalPlaces_), hasRoot_(rhs.hasRoot_) { + rhs.os_ = 0; + } +#endif + //! Reset the writer with a new stream. /*! This function reset the writer with a new stream and default settings, @@ -184,12 +193,14 @@ public: bool Double(double d) { Prefix(kNumberType); return EndValue(WriteDouble(d)); } bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); (void)copy; Prefix(kNumberType); return EndValue(WriteString(str, length)); } bool String(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); (void)copy; Prefix(kStringType); return EndValue(WriteString(str, length)); @@ -209,10 +220,18 @@ public: bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } +#if RAPIDJSON_HAS_STDSTRING + bool Key(const std::basic_string& str) + { + return Key(str.data(), SizeType(str.size())); + } +#endif + bool EndObject(SizeType memberCount = 0) { (void)memberCount; - RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); - RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); + RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); // not inside an Object + RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); // currently inside an Array, not Object + RAPIDJSON_ASSERT(0 == level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value level_stack_.template Pop(1); return EndValue(WriteEndObject()); } @@ -236,9 +255,9 @@ public: //@{ //! Simpler but slower overload. - bool String(const Ch* str) { return String(str, internal::StrLen(str)); } - bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } - + bool String(const Ch* const& str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* const& str) { return Key(str, internal::StrLen(str)); } + //@} //! Write a raw JSON value. @@ -249,7 +268,19 @@ public: \param length Length of the json. \param type Type of the root of json. */ - bool RawValue(const Ch* json, size_t length, Type type) { Prefix(type); return EndValue(WriteRawValue(json, length)); } + bool RawValue(const Ch* json, size_t length, Type type) { + RAPIDJSON_ASSERT(json != 0); + Prefix(type); + return EndValue(WriteRawValue(json, length)); + } + + //! Flush the output stream. + /*! + Allows the user to flush the output stream immediately. + */ + void Flush() { + os_->Flush(); + } protected: //! Information for each nested level @@ -283,7 +314,7 @@ protected: const char* end = internal::i32toa(i, buffer); PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); + PutUnsafe(*os_, static_cast(*p)); return true; } @@ -292,7 +323,7 @@ protected: const char* end = internal::u32toa(u, buffer); PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); + PutUnsafe(*os_, static_cast(*p)); return true; } @@ -301,7 +332,7 @@ protected: const char* end = internal::i64toa(i64, buffer); PutReserve(*os_, static_cast(end - buffer)); for (const char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); + PutUnsafe(*os_, static_cast(*p)); return true; } @@ -310,7 +341,7 @@ protected: char* end = internal::u64toa(u64, buffer); PutReserve(*os_, static_cast(end - buffer)); for (char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); + PutUnsafe(*os_, static_cast(*p)); return true; } @@ -338,12 +369,12 @@ protected: char* end = internal::dtoa(d, buffer, maxDecimalPlaces_); PutReserve(*os_, static_cast(end - buffer)); for (char* p = buffer; p != end; ++p) - PutUnsafe(*os_, static_cast(*p)); + PutUnsafe(*os_, static_cast(*p)); return true; } bool WriteString(const Ch* str, SizeType length) { - static const typename TargetEncoding::Ch hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + static const typename OutputStream::Ch hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; static const char escape[256] = { #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 //0 1 2 3 4 5 6 7 8 9 A B C D E F @@ -399,7 +430,7 @@ protected: else if ((sizeof(Ch) == 1 || static_cast(c) < 256) && RAPIDJSON_UNLIKELY(escape[static_cast(c)])) { is.Take(); PutUnsafe(*os_, '\\'); - PutUnsafe(*os_, static_cast(escape[static_cast(c)])); + PutUnsafe(*os_, static_cast(escape[static_cast(c)])); if (escape[static_cast(c)] == 'u') { PutUnsafe(*os_, '0'); PutUnsafe(*os_, '0'); @@ -427,9 +458,13 @@ protected: bool WriteRawValue(const Ch* json, size_t length) { PutReserve(*os_, length); - for (size_t i = 0; i < length; i++) { - RAPIDJSON_ASSERT(json[i] != '\0'); - PutUnsafe(*os_, json[i]); + GenericStringStream is(json); + while (RAPIDJSON_LIKELY(is.Tell() < length)) { + RAPIDJSON_ASSERT(is.Peek() != '\0'); + if (RAPIDJSON_UNLIKELY(!(writeFlags & kWriteValidateEncodingFlag ? + Transcoder::Validate(is, *os_) : + Transcoder::TranscodeUnsafe(is, *os_)))) + return false; } return true; } @@ -457,7 +492,7 @@ protected: // Flush the value if it is the top level one. bool EndValue(bool ret) { if (RAPIDJSON_UNLIKELY(level_stack_.Empty())) // end of json text - os_->Flush(); + Flush(); return ret; } @@ -561,7 +596,7 @@ inline bool Writer::ScanWriteUnescapedString(StringStream& is, siz // The rest of string using SIMD static const char dquote[16] = { '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"', '\"' }; static const char bslash[16] = { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' }; - static const char space[16] = { 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19, 0x19 }; + static const char space[16] = { 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F, 0x1F }; const __m128i dq = _mm_loadu_si128(reinterpret_cast(&dquote[0])); const __m128i bs = _mm_loadu_si128(reinterpret_cast(&bslash[0])); const __m128i sp = _mm_loadu_si128(reinterpret_cast(&space[0])); @@ -570,7 +605,7 @@ inline bool Writer::ScanWriteUnescapedString(StringStream& is, siz const __m128i s = _mm_load_si128(reinterpret_cast(p)); const __m128i t1 = _mm_cmpeq_epi8(s, dq); const __m128i t2 = _mm_cmpeq_epi8(s, bs); - const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x19) == 0x19 + const __m128i t3 = _mm_cmpeq_epi8(_mm_max_epu8(s, sp), sp); // s < 0x20 <=> max(s, 0x1F) == 0x1F const __m128i x = _mm_or_si128(_mm_or_si128(t1, t2), t3); unsigned short r = static_cast(_mm_movemask_epi8(x)); if (RAPIDJSON_UNLIKELY(r != 0)) { // some of characters is escaped @@ -595,15 +630,79 @@ inline bool Writer::ScanWriteUnescapedString(StringStream& is, siz is.src_ = p; return RAPIDJSON_LIKELY(is.Tell() < length); } -#endif // defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) +#elif defined(RAPIDJSON_NEON) +template<> +inline bool Writer::ScanWriteUnescapedString(StringStream& is, size_t length) { + if (length < 16) + return RAPIDJSON_LIKELY(is.Tell() < length); + + if (!RAPIDJSON_LIKELY(is.Tell() < length)) + return false; + + const char* p = is.src_; + const char* end = is.head_ + length; + const char* nextAligned = reinterpret_cast((reinterpret_cast(p) + 15) & static_cast(~15)); + const char* endAligned = reinterpret_cast(reinterpret_cast(end) & static_cast(~15)); + if (nextAligned > end) + return true; + + while (p != nextAligned) + if (*p < 0x20 || *p == '\"' || *p == '\\') { + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); + } + else + os_->PutUnsafe(*p++); + + // The rest of string using SIMD + const uint8x16_t s0 = vmovq_n_u8('"'); + const uint8x16_t s1 = vmovq_n_u8('\\'); + const uint8x16_t s2 = vmovq_n_u8('\b'); + const uint8x16_t s3 = vmovq_n_u8(32); + + for (; p != endAligned; p += 16) { + const uint8x16_t s = vld1q_u8(reinterpret_cast(p)); + uint8x16_t x = vceqq_u8(s, s0); + x = vorrq_u8(x, vceqq_u8(s, s1)); + x = vorrq_u8(x, vceqq_u8(s, s2)); + x = vorrq_u8(x, vcltq_u8(s, s3)); + + x = vrev64q_u8(x); // Rev in 64 + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + + SizeType len = 0; + bool escaped = false; + if (low == 0) { + if (high != 0) { + unsigned lz = (unsigned)__builtin_clzll(high); + len = 8 + (lz >> 3); + escaped = true; + } + } else { + unsigned lz = (unsigned)__builtin_clzll(low); + len = lz >> 3; + escaped = true; + } + if (RAPIDJSON_UNLIKELY(escaped)) { // some of characters is escaped + char* q = reinterpret_cast(os_->PushUnsafe(len)); + for (size_t i = 0; i < len; i++) + q[i] = p[i]; + + p += len; + break; + } + vst1q_u8(reinterpret_cast(os_->PushUnsafe(16)), s); + } + + is.src_ = p; + return RAPIDJSON_LIKELY(is.Tell() < length); +} +#endif // RAPIDJSON_NEON RAPIDJSON_NAMESPACE_END -#ifdef _MSC_VER -RAPIDJSON_DIAG_POP -#endif - -#ifdef __clang__ +#if defined(_MSC_VER) || defined(__clang__) RAPIDJSON_DIAG_POP #endif From a8e86c3530ae6376c7a752dd07c10d6cfe101967 Mon Sep 17 00:00:00 2001 From: XMRig Date: Sun, 4 Aug 2019 22:07:05 +0700 Subject: [PATCH 03/26] More compact JSON formatting. --- src/3rdparty/rapidjson/prettywriter.h | 12 ++++++++---- src/3rdparty/rapidjson/rapidjson.h | 2 +- src/3rdparty/rapidjson/writer.h | 1 + src/base/io/json/Json_unix.cpp | 2 ++ src/base/io/json/Json_win.cpp | 2 ++ src/base/net/http/HttpApiResponse.cpp | 2 ++ 6 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/3rdparty/rapidjson/prettywriter.h b/src/3rdparty/rapidjson/prettywriter.h index 45afb694..c7c29b21 100644 --- a/src/3rdparty/rapidjson/prettywriter.h +++ b/src/3rdparty/rapidjson/prettywriter.h @@ -164,9 +164,10 @@ public: (void)memberCount; RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); - bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + typename Base::Level* level = Base::level_stack_.template Pop(1); + bool empty = level->valueCount == 0; - if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { + if (!empty && !level->inLine) { Base::os_->Put('\n'); WriteIndent(); } @@ -211,13 +212,16 @@ protected: typename Base::Level* level = Base::level_stack_.template Top(); if (level->inArray) { + level->inLine = (formatOptions_ & kFormatSingleLineArray) && type != kObjectType && type != kArrayType; + if (level->valueCount > 0) { Base::os_->Put(','); // add comma if it is not the first element in array - if (formatOptions_ & kFormatSingleLineArray) + if (level->inLine) { Base::os_->Put(' '); + } } - if (!(formatOptions_ & kFormatSingleLineArray)) { + if (!level->inLine) { Base::os_->Put('\n'); WriteIndent(); } diff --git a/src/3rdparty/rapidjson/rapidjson.h b/src/3rdparty/rapidjson/rapidjson.h index 549936ff..78c8aae0 100644 --- a/src/3rdparty/rapidjson/rapidjson.h +++ b/src/3rdparty/rapidjson/rapidjson.h @@ -403,7 +403,7 @@ RAPIDJSON_NAMESPACE_END */ #ifndef RAPIDJSON_ASSERT #include -#define RAPIDJSON_ASSERT(x) assert(x) +#define RAPIDJSON_ASSERT(x) #endif // RAPIDJSON_ASSERT /////////////////////////////////////////////////////////////////////////////// diff --git a/src/3rdparty/rapidjson/writer.h b/src/3rdparty/rapidjson/writer.h index 6f5b6903..1d33b2f9 100644 --- a/src/3rdparty/rapidjson/writer.h +++ b/src/3rdparty/rapidjson/writer.h @@ -288,6 +288,7 @@ protected: Level(bool inArray_) : valueCount(0), inArray(inArray_) {} size_t valueCount; //!< number of values in this level bool inArray; //!< true if in array, otherwise in object + bool inLine = false; }; static const size_t kDefaultLevelDepth = 32; diff --git a/src/base/io/json/Json_unix.cpp b/src/base/io/json/Json_unix.cpp index eeef9564..dedea947 100644 --- a/src/base/io/json/Json_unix.cpp +++ b/src/base/io/json/Json_unix.cpp @@ -56,6 +56,8 @@ bool xmrig::Json::save(const char *fileName, const rapidjson::Document &doc) rapidjson::OStreamWrapper osw(ofs); rapidjson::PrettyWriter writer(osw); + writer.SetFormatOptions(rapidjson::kFormatSingleLineArray); + doc.Accept(writer); return true; diff --git a/src/base/io/json/Json_win.cpp b/src/base/io/json/Json_win.cpp index 0faccdea..73aff2c5 100644 --- a/src/base/io/json/Json_win.cpp +++ b/src/base/io/json/Json_win.cpp @@ -118,6 +118,8 @@ bool xmrig::Json::save(const char *fileName, const rapidjson::Document &doc) OStreamWrapper osw(ofs); PrettyWriter writer(osw); + writer.SetFormatOptions(kFormatSingleLineArray); + doc.Accept(writer); return true; diff --git a/src/base/net/http/HttpApiResponse.cpp b/src/base/net/http/HttpApiResponse.cpp index bf91445a..5fe92636 100644 --- a/src/base/net/http/HttpApiResponse.cpp +++ b/src/base/net/http/HttpApiResponse.cpp @@ -80,6 +80,8 @@ void xmrig::HttpApiResponse::end() StringBuffer buffer(nullptr, 4096); PrettyWriter writer(buffer); writer.SetMaxDecimalPlaces(10); + writer.SetFormatOptions(kFormatSingleLineArray); + m_doc.Accept(writer); HttpResponse::end(buffer.GetString(), buffer.GetSize()); From 044fbd7e8289f65e89cd07df3f048e854616b847 Mon Sep 17 00:00:00 2001 From: XMRig Date: Mon, 5 Aug 2019 16:00:48 +0700 Subject: [PATCH 04/26] Changed init signature. --- src/crypto/rx/RxCache.cpp | 4 ++-- src/crypto/rx/RxCache.h | 4 ++-- src/crypto/rx/RxDataset.cpp | 2 +- src/crypto/rx/RxDataset.h | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/crypto/rx/RxCache.cpp b/src/crypto/rx/RxCache.cpp index a5e9efb3..13ff0cd3 100644 --- a/src/crypto/rx/RxCache.cpp +++ b/src/crypto/rx/RxCache.cpp @@ -62,7 +62,7 @@ xmrig::RxCache::~RxCache() } -bool xmrig::RxCache::init(const void *seed) +bool xmrig::RxCache::init(const uint8_t *seed) { if (isReady(seed)) { return false; @@ -75,7 +75,7 @@ bool xmrig::RxCache::init(const void *seed) } -bool xmrig::RxCache::isReady(const void *seed) const +bool xmrig::RxCache::isReady(const uint8_t *seed) const { return memcmp(m_seed, seed, sizeof(m_seed)) == 0; } diff --git a/src/crypto/rx/RxCache.h b/src/crypto/rx/RxCache.h index 80c1faba..433058b7 100644 --- a/src/crypto/rx/RxCache.h +++ b/src/crypto/rx/RxCache.h @@ -52,12 +52,12 @@ public: inline const uint8_t *seed() const { return m_seed; } inline randomx_cache *get() const { return m_cache; } - bool init(const void *seed); + bool init(const uint8_t *seed); static inline constexpr size_t size() { return RANDOMX_CACHE_MAX_SIZE; } private: - bool isReady(const void *seed) const; + bool isReady(const uint8_t *seed) const; int m_flags = 0; randomx_cache *m_cache = nullptr; diff --git a/src/crypto/rx/RxDataset.cpp b/src/crypto/rx/RxDataset.cpp index 7d498c4d..50459a55 100644 --- a/src/crypto/rx/RxDataset.cpp +++ b/src/crypto/rx/RxDataset.cpp @@ -64,7 +64,7 @@ xmrig::RxDataset::~RxDataset() } -bool xmrig::RxDataset::init(const void *seed, uint32_t numThreads) +bool xmrig::RxDataset::init(const uint8_t *seed, uint32_t numThreads) { cache()->init(seed); diff --git a/src/crypto/rx/RxDataset.h b/src/crypto/rx/RxDataset.h index d3488668..932f4ed9 100644 --- a/src/crypto/rx/RxDataset.h +++ b/src/crypto/rx/RxDataset.h @@ -52,7 +52,7 @@ public: inline randomx_dataset *get() const { return m_dataset; } inline RxCache *cache() const { return m_cache; } - bool init(const void *seed, uint32_t numThreads); + bool init(const uint8_t *seed, uint32_t numThreads); std::pair hugePages() const; static inline constexpr size_t size() { return RANDOMX_DATASET_MAX_SIZE; } From 3543abcc3cb99155b3620b53daa304daae373b2a Mon Sep 17 00:00:00 2001 From: XMRig Date: Mon, 5 Aug 2019 17:48:12 +0700 Subject: [PATCH 05/26] Fixed crash and added error message about seed_hash field. --- src/base/net/stratum/Client.cpp | 13 +++++++++++-- src/crypto/rx/RxCache.cpp | 2 +- src/crypto/rx/RxCache.h | 2 ++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/base/net/stratum/Client.cpp b/src/base/net/stratum/Client.cpp index 618e132c..fe182336 100644 --- a/src/base/net/stratum/Client.cpp +++ b/src/base/net/stratum/Client.cpp @@ -335,13 +335,19 @@ bool xmrig::Client::parseJob(const rapidjson::Value ¶ms, int *code) job.setAlgorithm(algo); } - job.setSeedHash(Json::getString(params, "seed_hash")); job.setHeight(Json::getUint64(params, "height")); if (!verifyAlgorithm(job.algorithm(), algo)) { *code = 6; + return false; + } - close(); + if (job.algorithm().family() == Algorithm::RANDOM_X && !job.setSeedHash(Json::getString(params, "seed_hash"))) { + if (!isQuiet()) { + LOG_ERR("[%s] failed to parse field \"seed_hash\" required by RandomX", url(), algo); + } + + *code = 7; return false; } @@ -693,6 +699,9 @@ void xmrig::Client::parseNotification(const char *method, const rapidjson::Value if (parseJob(params, &code)) { m_listener->onJobReceived(this, m_job, params); } + else { + close(); + } return; } diff --git a/src/crypto/rx/RxCache.cpp b/src/crypto/rx/RxCache.cpp index 13ff0cd3..985f4bf3 100644 --- a/src/crypto/rx/RxCache.cpp +++ b/src/crypto/rx/RxCache.cpp @@ -77,5 +77,5 @@ bool xmrig::RxCache::init(const uint8_t *seed) bool xmrig::RxCache::isReady(const uint8_t *seed) const { - return memcmp(m_seed, seed, sizeof(m_seed)) == 0; + return m_initCount && memcmp(m_seed, seed, sizeof(m_seed)) == 0; } diff --git a/src/crypto/rx/RxCache.h b/src/crypto/rx/RxCache.h index 433058b7..e6b2397c 100644 --- a/src/crypto/rx/RxCache.h +++ b/src/crypto/rx/RxCache.h @@ -51,6 +51,7 @@ public: inline bool isJIT() const { return m_flags & 8; } inline const uint8_t *seed() const { return m_seed; } inline randomx_cache *get() const { return m_cache; } + inline uint64_t initCount() const { return m_initCount; } bool init(const uint8_t *seed); @@ -61,6 +62,7 @@ private: int m_flags = 0; randomx_cache *m_cache = nullptr; + uint64_t m_initCount = 0; uint8_t m_seed[32]; }; From 4afc987111652b3116518c9eb9ac5d4ba2dd7821 Mon Sep 17 00:00:00 2001 From: XMRig Date: Mon, 5 Aug 2019 18:20:56 +0700 Subject: [PATCH 06/26] Fix for previous commit. --- src/crypto/rx/RxCache.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/crypto/rx/RxCache.cpp b/src/crypto/rx/RxCache.cpp index 985f4bf3..92c366a2 100644 --- a/src/crypto/rx/RxCache.cpp +++ b/src/crypto/rx/RxCache.cpp @@ -71,6 +71,8 @@ bool xmrig::RxCache::init(const uint8_t *seed) memcpy(m_seed, seed, sizeof(m_seed)); randomx_init_cache(m_cache, m_seed, sizeof(m_seed)); + m_initCount++; + return true; } From 66d8598f9f6e0315367dda36999180b3b78c909a Mon Sep 17 00:00:00 2001 From: XMRig Date: Mon, 5 Aug 2019 21:55:52 +0700 Subject: [PATCH 07/26] #1092 Fixed crash if use wrong affinity on NUMA systems. --- src/crypto/common/VirtualMemory.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/crypto/common/VirtualMemory.cpp b/src/crypto/common/VirtualMemory.cpp index 5f7c4551..081b6c0f 100644 --- a/src/crypto/common/VirtualMemory.cpp +++ b/src/crypto/common/VirtualMemory.cpp @@ -62,15 +62,18 @@ uint32_t xmrig::VirtualMemory::bindToNUMANode(int64_t affinity) LOG_WARN("CPU #%02u warning: \"can't bind memory\"", puId); } - hwloc_obj_t node = nullptr; - uint32_t nodeId = 0; + uint32_t nodeId = 0; - while ((node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node)) != nullptr) { - if (hwloc_bitmap_intersects(node->cpuset, pu->cpuset)) { - nodeId = node->os_index; + if (pu) { + hwloc_obj_t node = nullptr; - break; - } + while ((node = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NUMANODE, node)) != nullptr) { + if (hwloc_bitmap_intersects(node->cpuset, pu->cpuset)) { + nodeId = node->os_index; + + break; + } + } } hwloc_topology_destroy(topology); From d4c5e414c27087161c7e8c7936a049ef03a0319e Mon Sep 17 00:00:00 2001 From: XMRig Date: Tue, 6 Aug 2019 12:28:37 +0700 Subject: [PATCH 08/26] Added hwloc topology examples. --- doc/topology/AMD_FX_8320_windows_2_0_4.xml | 86 +++ ...AMD_Opteron_6272_x4_N8_linux_2_0_4_LXC.xml | 234 ++++++ .../AMD_Opteron_6278_x2_UMA_windows_2_0_4.xml | 294 ++++++++ .../AMD_Opteron_6348_x4_N8_linux_1_11_2.xml | 550 ++++++++++++++ .../AMD_Opteron_6380_x4_N8_linux_1_11_5.xml | 670 ++++++++++++++++++ .../AMD_Ryzen_7_2700X_windows_2_0_4.xml | 105 +++ ...yzen_Threadripper_2950X_N2_linux_2_0_4.xml | 226 ++++++ ...en_Threadripper_2950X_UMA_linux_1_11_9.xml | 328 +++++++++ .../Intel_Core_i7-3770_linux_2_0_4.xml | 87 +++ .../Intel_Core_i7-6700_linux_2_0_4.xml | 88 +++ .../Intel_Core_i7-6700_windows_2_0_4.xml | 61 ++ .../Intel_Core_i7-7660U_windows_2_0_4.xml | 41 ++ ...tel_Xeon_E5-4650_0_x4_N4_windows_2_0_4.xml | 477 +++++++++++++ .../Intel_Xeon_E5620_x2_UMA_windows_2_0_4.xml | 111 +++ ...Intel_Xeon_E7-4870_x4_N4_windows_2_0_4.xml | 541 ++++++++++++++ ...el_Xeon_Silver_4114_x2_N2_linux_1_11_9.xml | 403 +++++++++++ ...tel_Xeon_Silver_4114_x2_N2_linux_2_0_4.xml | 263 +++++++ 17 files changed, 4565 insertions(+) create mode 100644 doc/topology/AMD_FX_8320_windows_2_0_4.xml create mode 100644 doc/topology/AMD_Opteron_6272_x4_N8_linux_2_0_4_LXC.xml create mode 100644 doc/topology/AMD_Opteron_6278_x2_UMA_windows_2_0_4.xml create mode 100644 doc/topology/AMD_Opteron_6348_x4_N8_linux_1_11_2.xml create mode 100644 doc/topology/AMD_Opteron_6380_x4_N8_linux_1_11_5.xml create mode 100644 doc/topology/AMD_Ryzen_7_2700X_windows_2_0_4.xml create mode 100644 doc/topology/AMD_Ryzen_Threadripper_2950X_N2_linux_2_0_4.xml create mode 100644 doc/topology/AMD_Ryzen_Threadripper_2950X_UMA_linux_1_11_9.xml create mode 100644 doc/topology/Intel_Core_i7-3770_linux_2_0_4.xml create mode 100644 doc/topology/Intel_Core_i7-6700_linux_2_0_4.xml create mode 100644 doc/topology/Intel_Core_i7-6700_windows_2_0_4.xml create mode 100644 doc/topology/Intel_Core_i7-7660U_windows_2_0_4.xml create mode 100644 doc/topology/Intel_Xeon_E5-4650_0_x4_N4_windows_2_0_4.xml create mode 100644 doc/topology/Intel_Xeon_E5620_x2_UMA_windows_2_0_4.xml create mode 100644 doc/topology/Intel_Xeon_E7-4870_x4_N4_windows_2_0_4.xml create mode 100644 doc/topology/Intel_Xeon_Silver_4114_x2_N2_linux_1_11_9.xml create mode 100644 doc/topology/Intel_Xeon_Silver_4114_x2_N2_linux_2_0_4.xml diff --git a/doc/topology/AMD_FX_8320_windows_2_0_4.xml b/doc/topology/AMD_FX_8320_windows_2_0_4.xml new file mode 100644 index 00000000..55fe6b5d --- /dev/null +++ b/doc/topology/AMD_FX_8320_windows_2_0_4.xml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/AMD_Opteron_6272_x4_N8_linux_2_0_4_LXC.xml b/doc/topology/AMD_Opteron_6272_x4_N8_linux_2_0_4_LXC.xml new file mode 100644 index 00000000..39576bb4 --- /dev/null +++ b/doc/topology/AMD_Opteron_6272_x4_N8_linux_2_0_4_LXC.xml @@ -0,0 +1,234 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 1 2 3 4 5 6 7 + 10 16 16 22 16 22 16 22 16 10 + 22 16 22 16 22 16 16 22 10 16 + 16 22 16 22 22 16 16 10 22 16 + 22 16 16 22 16 22 10 16 16 22 + 22 16 22 16 16 10 22 16 16 22 + 16 22 16 22 10 16 22 16 22 16 + 22 16 16 10 + + \ No newline at end of file diff --git a/doc/topology/AMD_Opteron_6278_x2_UMA_windows_2_0_4.xml b/doc/topology/AMD_Opteron_6278_x2_UMA_windows_2_0_4.xml new file mode 100644 index 00000000..b59f773c --- /dev/null +++ b/doc/topology/AMD_Opteron_6278_x2_UMA_windows_2_0_4.xml @@ -0,0 +1,294 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/AMD_Opteron_6348_x4_N8_linux_1_11_2.xml b/doc/topology/AMD_Opteron_6348_x4_N8_linux_1_11_2.xml new file mode 100644 index 00000000..2d889819 --- /dev/null +++ b/doc/topology/AMD_Opteron_6348_x4_N8_linux_1_11_2.xml @@ -0,0 +1,550 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/AMD_Opteron_6380_x4_N8_linux_1_11_5.xml b/doc/topology/AMD_Opteron_6380_x4_N8_linux_1_11_5.xml new file mode 100644 index 00000000..2ecbe3cb --- /dev/null +++ b/doc/topology/AMD_Opteron_6380_x4_N8_linux_1_11_5.xml @@ -0,0 +1,670 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/AMD_Ryzen_7_2700X_windows_2_0_4.xml b/doc/topology/AMD_Ryzen_7_2700X_windows_2_0_4.xml new file mode 100644 index 00000000..e3ecb6fd --- /dev/null +++ b/doc/topology/AMD_Ryzen_7_2700X_windows_2_0_4.xml @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/AMD_Ryzen_Threadripper_2950X_N2_linux_2_0_4.xml b/doc/topology/AMD_Ryzen_Threadripper_2950X_N2_linux_2_0_4.xml new file mode 100644 index 00000000..c168e2a0 --- /dev/null +++ b/doc/topology/AMD_Ryzen_Threadripper_2950X_N2_linux_2_0_4.xml @@ -0,0 +1,226 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 1 + 10 16 16 10 + + diff --git a/doc/topology/AMD_Ryzen_Threadripper_2950X_UMA_linux_1_11_9.xml b/doc/topology/AMD_Ryzen_Threadripper_2950X_UMA_linux_1_11_9.xml new file mode 100644 index 00000000..ed3776c0 --- /dev/null +++ b/doc/topology/AMD_Ryzen_Threadripper_2950X_UMA_linux_1_11_9.xml @@ -0,0 +1,328 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/Intel_Core_i7-3770_linux_2_0_4.xml b/doc/topology/Intel_Core_i7-3770_linux_2_0_4.xml new file mode 100644 index 00000000..18c80210 --- /dev/null +++ b/doc/topology/Intel_Core_i7-3770_linux_2_0_4.xml @@ -0,0 +1,87 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/Intel_Core_i7-6700_linux_2_0_4.xml b/doc/topology/Intel_Core_i7-6700_linux_2_0_4.xml new file mode 100644 index 00000000..c80c7403 --- /dev/null +++ b/doc/topology/Intel_Core_i7-6700_linux_2_0_4.xml @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/Intel_Core_i7-6700_windows_2_0_4.xml b/doc/topology/Intel_Core_i7-6700_windows_2_0_4.xml new file mode 100644 index 00000000..dd3c201c --- /dev/null +++ b/doc/topology/Intel_Core_i7-6700_windows_2_0_4.xml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/Intel_Core_i7-7660U_windows_2_0_4.xml b/doc/topology/Intel_Core_i7-7660U_windows_2_0_4.xml new file mode 100644 index 00000000..01a29e86 --- /dev/null +++ b/doc/topology/Intel_Core_i7-7660U_windows_2_0_4.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/Intel_Xeon_E5-4650_0_x4_N4_windows_2_0_4.xml b/doc/topology/Intel_Xeon_E5-4650_0_x4_N4_windows_2_0_4.xml new file mode 100644 index 00000000..7811a49f --- /dev/null +++ b/doc/topology/Intel_Xeon_E5-4650_0_x4_N4_windows_2_0_4.xml @@ -0,0 +1,477 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/Intel_Xeon_E5620_x2_UMA_windows_2_0_4.xml b/doc/topology/Intel_Xeon_E5620_x2_UMA_windows_2_0_4.xml new file mode 100644 index 00000000..9dad397a --- /dev/null +++ b/doc/topology/Intel_Xeon_E5620_x2_UMA_windows_2_0_4.xml @@ -0,0 +1,111 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/Intel_Xeon_E7-4870_x4_N4_windows_2_0_4.xml b/doc/topology/Intel_Xeon_E7-4870_x4_N4_windows_2_0_4.xml new file mode 100644 index 00000000..3d0a6736 --- /dev/null +++ b/doc/topology/Intel_Xeon_E7-4870_x4_N4_windows_2_0_4.xml @@ -0,0 +1,541 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/Intel_Xeon_Silver_4114_x2_N2_linux_1_11_9.xml b/doc/topology/Intel_Xeon_Silver_4114_x2_N2_linux_1_11_9.xml new file mode 100644 index 00000000..0dadfed2 --- /dev/null +++ b/doc/topology/Intel_Xeon_Silver_4114_x2_N2_linux_1_11_9.xml @@ -0,0 +1,403 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/topology/Intel_Xeon_Silver_4114_x2_N2_linux_2_0_4.xml b/doc/topology/Intel_Xeon_Silver_4114_x2_N2_linux_2_0_4.xml new file mode 100644 index 00000000..fd56a10c --- /dev/null +++ b/doc/topology/Intel_Xeon_Silver_4114_x2_N2_linux_2_0_4.xml @@ -0,0 +1,263 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 1 + 10 21 21 10 + + From 46f1661fd905e6c8634ef60a7c55a94744fbcef0 Mon Sep 17 00:00:00 2001 From: xmrig Date: Tue, 6 Aug 2019 12:48:00 +0700 Subject: [PATCH 09/26] Update API.md --- doc/API.md | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/doc/API.md b/doc/API.md index 3357eabb..2cd0fbbe 100644 --- a/doc/API.md +++ b/doc/API.md @@ -1,26 +1,39 @@ # HTTP API -If you want use API you need choice a port where is internal HTTP server will listen for incoming connections. API will not available if miner built without `libmicrohttpd`. +If you want use HTTP API you need enable it (`"enabled": true,`) then choice `port` and optionaly `host`. API not available if miner built without HTTP support (`-DWITH_HTTP=OFF`). + +Offical HTTP client for API: http://workers.xmrig.info/ Example configuration: ```json "api": { - "port": 44444, - "access-token": "TOKEN", - "worker-id": null, - "ipv6": false, - "restricted": false + "id": null, + "worker-id": null, }, +"http": { + "enabled": false, + "host": "127.0.0.1", + "port": 0, + "access-token": null, + "restricted": true +} ``` -* **port** Port for incoming connections `http://:`. -* **access-token** [Bearer](https://gist.github.com/xmrig/c75fdd1f8e0f3bac05500be2ab718f8e#file-api-html-L54) access token to secure access to API. +#### Global API options +* **id** Miner ID, if not set created automatically. * **worker-id** Optional worker name, if not set will be detected automatically. -* **ipv6** Enable (`true`) or disable (`false`) IPv6 for API. + +#### HTTP API options, +* **enabled** Enable (`true`) or disable (`false`) HTTP API. +* **host** Host for incoming connections `http://:`, to allow connections from all interfaces use `0.0.0.0` (IPv4) or `::` (IPv4+IPv6). +* **port** Port for incoming connections `http://:`, zero port is valid option and means random port. +* **access-token** [Bearer](https://gist.github.com/xmrig/c75fdd1f8e0f3bac05500be2ab718f8e#file-api-html-L54) access token to secure access to API. Miner support this token only via `Authorization` header. * **restricted** Use `false` to allow remote configuration. -If you prefer use command line options instead of config file, you can use options: `--api-port`, `--api-access-token`, `--api-worker-id`, `--api-ipv6` and `api-no-restricted`. +If you prefer use command line options instead of config file, you can use options: `--api-id`, `--api-worker-id`, `--http-enabled`, `--http-host`, `--http-access-token`, `--http-port`, `--http-no-restricted`. + +Versions before 2.15 was use another options for API https://github.com/xmrig/xmrig/issues/1007 ## Endpoints @@ -50,4 +63,4 @@ Curl example: ``` curl -v --data-binary @config.json -X PUT -H "Content-Type: application/json" -H "Authorization: Bearer SECRET" http://127.0.0.1:44444/1/config -``` \ No newline at end of file +``` From 3c5cb142cb10c1a24d5209c135e1690f76066150 Mon Sep 17 00:00:00 2001 From: XMRig Date: Tue, 6 Aug 2019 13:21:39 +0700 Subject: [PATCH 10/26] Added "/2/config" alias for config API. --- src/base/kernel/Base.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/base/kernel/Base.cpp b/src/base/kernel/Base.cpp index d290e7f4..03feef89 100644 --- a/src/base/kernel/Base.cpp +++ b/src/base/kernel/Base.cpp @@ -57,6 +57,14 @@ #endif +namespace xmrig { + +static const char *kConfigPathV1 = "/1/config"; +static const char *kConfigPathV2 = "/2/config"; + +} // namespace xmrig + + class xmrig::BasePrivate { public: @@ -296,7 +304,7 @@ void xmrig::Base::onFileChanged(const String &fileName) void xmrig::Base::onRequest(IApiRequest &request) { if (request.method() == IApiRequest::METHOD_GET) { - if (request.url() == "/1/config") { + if (request.url() == kConfigPathV1 || request.url() == kConfigPathV2) { if (request.isRestricted()) { return request.done(403); } @@ -306,7 +314,7 @@ void xmrig::Base::onRequest(IApiRequest &request) } } else if (request.method() == IApiRequest::METHOD_PUT || request.method() == IApiRequest::METHOD_POST) { - if (request.url() == "/1/config") { + if (request.url() == kConfigPathV1 || request.url() == kConfigPathV2) { request.accept(); if (!reload(request.json())) { From cc4351d49fe7b2cabf8a9356d5e42ae302a43119 Mon Sep 17 00:00:00 2001 From: xmrig Date: Tue, 6 Aug 2019 13:23:51 +0700 Subject: [PATCH 11/26] Update CPU.md --- doc/CPU.md | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/doc/CPU.md b/doc/CPU.md index b93651f4..f5d07745 100644 --- a/doc/CPU.md +++ b/doc/CPU.md @@ -22,16 +22,8 @@ Example below demonstrate all primary ideas of flexible profiles configuration: "hw-aes": null, "priority": null, "asm": true, - "rx/wow": [ - -1, - -1, - -1, - -1, - ], - "cn": [ - 0, - 2 - ], + "rx/wow": [-1, -1, -1, -1], + "cn": [0, 2], "cn-lite": [ { "intensity": 2, @@ -42,13 +34,8 @@ Example below demonstrate all primary ideas of flexible profiles configuration: "affinity": 2 } ], - "custom-profile": [ - 0, - 2, - ], - "*": [ - -1 - ], + "custom-profile": [0, 2], + "*": [-1], "cn/r": "custom-profile", "cn/0": false } From 0adab95ce424c48e53876e8e33c38116fe5359e6 Mon Sep 17 00:00:00 2001 From: xmrig Date: Wed, 7 Aug 2019 00:10:47 +0700 Subject: [PATCH 12/26] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97216550..87cd1d4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# v2.99.5-beta +- [#1066](https://github.com/xmrig/xmrig/issues/1066#issuecomment-518080529) Fixed crash and added error message if pool not ready for RandomX. +- [#1092](https://github.com/xmrig/xmrig/issues/1092) Fixed crash if wrong CPU affinity used. +- JSON arrays in config and API now more compact, single line if possible. + # v2.99.4-beta - [#1062](https://github.com/xmrig/xmrig/issues/1062) Fixed 32 bit support. **32 bit is slow and deprecated**. - [#1088](https://github.com/xmrig/xmrig/pull/1088) Fixed macOS compilation. From 96fd7545d1e37a0c89d0917ad35e2eb5d9b87e49 Mon Sep 17 00:00:00 2001 From: XMRig Date: Wed, 7 Aug 2019 16:13:23 +0700 Subject: [PATCH 13/26] Added class CpuThreads. --- src/backend/common/Threads.cpp | 27 +++------- src/backend/common/Threads.h | 8 +-- src/backend/cpu/CpuBackend.cpp | 4 +- src/backend/cpu/CpuConfig.cpp | 10 ++-- src/backend/cpu/CpuConfig.h | 16 +++--- src/backend/cpu/CpuThread.h | 12 ++--- src/backend/cpu/CpuThreads.cpp | 54 +++++++++++++++++++ src/backend/cpu/CpuThreads.h | 63 +++++++++++++++++++++++ src/backend/cpu/cpu.cmake | 2 + src/backend/cpu/interfaces/ICpuInfo.h | 2 +- src/backend/cpu/platform/BasicCpuInfo.cpp | 14 ++--- src/backend/cpu/platform/HwlocCpuInfo.cpp | 6 +-- 12 files changed, 160 insertions(+), 58 deletions(-) create mode 100644 src/backend/cpu/CpuThreads.cpp create mode 100644 src/backend/cpu/CpuThreads.h diff --git a/src/backend/common/Threads.cpp b/src/backend/common/Threads.cpp index 894c404b..323352d1 100644 --- a/src/backend/common/Threads.cpp +++ b/src/backend/common/Threads.cpp @@ -24,7 +24,7 @@ #include "backend/common/Threads.h" -#include "backend/cpu/CpuThread.h" +#include "backend/cpu/CpuThreads.h" #include "rapidjson/document.h" @@ -38,9 +38,9 @@ static const char *kAsterisk = "*"; template -const std::vector &xmrig::Threads::get(const String &profileName) const +const T &xmrig::Threads::get(const String &profileName) const { - static std::vector empty; + static T empty; if (profileName.isNull() || !has(profileName)) { return empty; } @@ -56,16 +56,9 @@ size_t xmrig::Threads::read(const rapidjson::Value &value) for (auto &member : value.GetObject()) { if (member.value.IsArray()) { - std::vector threads; + T threads(member.value); - for (auto &v : member.value.GetArray()) { - T thread(v); - if (thread.isValid()) { - threads.push_back(std::move(thread)); - } - } - - if (!threads.empty()) { + if (!threads.isEmpty()) { move(member.name.GetString(), std::move(threads)); } @@ -138,13 +131,7 @@ void xmrig::Threads::toJSON(rapidjson::Value &out, rapidjson::Document &doc) auto &allocator = doc.GetAllocator(); for (const auto &kv : m_profiles) { - Value arr(kArrayType); - - for (const T &thread : kv.second) { - arr.PushBack(thread.toJSON(doc), allocator); - } - - out.AddMember(kv.first.toJSON(), arr, allocator); + out.AddMember(kv.first.toJSON(), kv.second.toJSON(doc), allocator); } for (const Algorithm &algo : m_disabled) { @@ -159,6 +146,6 @@ void xmrig::Threads::toJSON(rapidjson::Value &out, rapidjson::Document &doc) namespace xmrig { -template class Threads; +template class Threads; } // namespace xmrig diff --git a/src/backend/common/Threads.h b/src/backend/common/Threads.h index bc9e36fd..2cb333d6 100644 --- a/src/backend/common/Threads.h +++ b/src/backend/common/Threads.h @@ -45,18 +45,18 @@ public: inline bool has(const char *profile) const { return m_profiles.count(profile) > 0; } inline bool isDisabled(const Algorithm &algo) const { return m_disabled.count(algo) > 0; } inline bool isExist(const Algorithm &algo) const { return isDisabled(algo) || m_aliases.count(algo) > 0 || has(algo.shortName()); } - inline const std::vector &get(const Algorithm &algo, bool strict = false) const { return get(profileName(algo, strict)); } + inline const T &get(const Algorithm &algo, bool strict = false) const { return get(profileName(algo, strict)); } inline void disable(const Algorithm &algo) { m_disabled.insert(algo); } - inline void move(const char *profile, std::vector &&threads) { m_profiles.insert({ profile, threads }); } + inline void move(const char *profile, T &&threads) { m_profiles.insert({ profile, threads }); } - const std::vector &get(const String &profileName) const; + const T &get(const String &profileName) const; size_t read(const rapidjson::Value &value); String profileName(const Algorithm &algorithm, bool strict = false) const; void toJSON(rapidjson::Value &out, rapidjson::Document &doc) const; private: std::map m_aliases; - std::map > m_profiles; + std::map m_profiles; std::set m_disabled; }; diff --git a/src/backend/cpu/CpuBackend.cpp b/src/backend/cpu/CpuBackend.cpp index 60ca8cf3..7f208c38 100644 --- a/src/backend/cpu/CpuBackend.cpp +++ b/src/backend/cpu/CpuBackend.cpp @@ -46,7 +46,7 @@ namespace xmrig { -extern template class Threads; +extern template class Threads; static const char *tag = CYAN_BG_BOLD(" cpu "); @@ -150,7 +150,7 @@ bool xmrig::CpuBackend::isEnabled() const bool xmrig::CpuBackend::isEnabled(const Algorithm &algorithm) const { - return !d_ptr->controller->config()->cpu().threads().get(algorithm).empty(); + return !d_ptr->controller->config()->cpu().threads().get(algorithm).isEmpty(); } diff --git a/src/backend/cpu/CpuConfig.cpp b/src/backend/cpu/CpuConfig.cpp index 582649ef..883ef506 100644 --- a/src/backend/cpu/CpuConfig.cpp +++ b/src/backend/cpu/CpuConfig.cpp @@ -62,7 +62,7 @@ static const char *kRx = "rx"; static const char *kRxWOW = "rx/wow"; #endif -extern template class Threads; +extern template class Threads; } @@ -103,15 +103,15 @@ rapidjson::Value xmrig::CpuConfig::toJSON(rapidjson::Document &doc) const std::vector xmrig::CpuConfig::get(const Miner *miner, const Algorithm &algorithm) const { std::vector out; - const std::vector &threads = m_threads.get(algorithm); + const CpuThreads &threads = m_threads.get(algorithm); - if (threads.empty()) { + if (threads.isEmpty()) { return out; } - out.reserve(threads.size()); + out.reserve(threads.count()); - for (const CpuThread &thread : threads) { + for (const CpuThread &thread : threads.data()) { out.push_back(CpuLaunchData(miner, algorithm, *this, thread)); } diff --git a/src/backend/cpu/CpuConfig.h b/src/backend/cpu/CpuConfig.h index 5b2f3f86..5aca5188 100644 --- a/src/backend/cpu/CpuConfig.h +++ b/src/backend/cpu/CpuConfig.h @@ -28,7 +28,7 @@ #include "backend/common/Threads.h" #include "backend/cpu/CpuLaunchData.h" -#include "backend/cpu/CpuThread.h" +#include "backend/cpu/CpuThreads.h" #include "crypto/common/Assembly.h" @@ -51,12 +51,12 @@ public: std::vector get(const Miner *miner, const Algorithm &algorithm) const; void read(const rapidjson::Value &value); - inline bool isEnabled() const { return m_enabled; } - inline bool isHugePages() const { return m_hugePages; } - inline bool isShouldSave() const { return m_shouldSave; } - inline const Assembly &assembly() const { return m_assembly; } - inline const Threads &threads() const { return m_threads; } - inline int priority() const { return m_priority; } + inline bool isEnabled() const { return m_enabled; } + inline bool isHugePages() const { return m_hugePages; } + inline bool isShouldSave() const { return m_shouldSave; } + inline const Assembly &assembly() const { return m_assembly; } + inline const Threads &threads() const { return m_threads; } + inline int priority() const { return m_priority; } private: void generate(); @@ -70,7 +70,7 @@ private: bool m_hugePages = true; bool m_shouldSave = false; int m_priority = -1; - Threads m_threads; + Threads m_threads; }; diff --git a/src/backend/cpu/CpuThread.h b/src/backend/cpu/CpuThread.h index adaffa68..7c7ce4be 100644 --- a/src/backend/cpu/CpuThread.h +++ b/src/backend/cpu/CpuThread.h @@ -22,11 +22,8 @@ * along with this program. If not, see . */ -#ifndef XMRIG_CPUTHREADCONFIG_H -#define XMRIG_CPUTHREADCONFIG_H - - -#include +#ifndef XMRIG_CPUTHREAD_H +#define XMRIG_CPUTHREAD_H #include "rapidjson/fwd.h" @@ -58,10 +55,7 @@ private: }; -typedef std::vector CpuThreads; - - } /* namespace xmrig */ -#endif /* XMRIG_CPUTHREADCONFIG_H */ +#endif /* XMRIG_CPUTHREAD_H */ diff --git a/src/backend/cpu/CpuThreads.cpp b/src/backend/cpu/CpuThreads.cpp new file mode 100644 index 00000000..f877c9d3 --- /dev/null +++ b/src/backend/cpu/CpuThreads.cpp @@ -0,0 +1,54 @@ +/* XMRig + * Copyright 2010 Jeff Garzik + * Copyright 2012-2014 pooler + * Copyright 2014 Lucas Jones + * Copyright 2014-2016 Wolf9466 + * Copyright 2016 Jay D Dee + * Copyright 2017-2018 XMR-Stak , + * Copyright 2018-2019 SChernykh + * Copyright 2016-2019 XMRig , + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#include "backend/cpu/CpuThreads.h" +#include "rapidjson/document.h" + + +xmrig::CpuThreads::CpuThreads(const rapidjson::Value &value) +{ + if (value.IsArray()) { + for (auto &v : value.GetArray()) { + CpuThread thread(v); + if (thread.isValid()) { + add(std::move(thread)); + } + } + } +} + + +rapidjson::Value xmrig::CpuThreads::toJSON(rapidjson::Document &doc) const +{ + using namespace rapidjson; + auto &allocator = doc.GetAllocator(); + + Value array(kArrayType); + for (const CpuThread &thread : m_data) { + array.PushBack(thread.toJSON(doc), allocator); + } + + return array; +} diff --git a/src/backend/cpu/CpuThreads.h b/src/backend/cpu/CpuThreads.h new file mode 100644 index 00000000..3951c202 --- /dev/null +++ b/src/backend/cpu/CpuThreads.h @@ -0,0 +1,63 @@ +/* XMRig + * Copyright 2010 Jeff Garzik + * Copyright 2012-2014 pooler + * Copyright 2014 Lucas Jones + * Copyright 2014-2016 Wolf9466 + * Copyright 2016 Jay D Dee + * Copyright 2017-2018 XMR-Stak , + * Copyright 2018-2019 SChernykh + * Copyright 2016-2019 XMRig , + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef XMRIG_CPUTHREADS_H +#define XMRIG_CPUTHREADS_H + + +#include + + +#include "backend/cpu/CpuThread.h" + + +namespace xmrig { + + +class CpuThreads +{ +public: + inline CpuThreads() {} + inline CpuThreads(size_t count) : m_data(count) {} + + CpuThreads(const rapidjson::Value &value); + + inline bool isEmpty() const { return m_data.empty(); } + inline const std::vector &data() const { return m_data; } + inline size_t count() const { return m_data.size(); } + inline void add(CpuThread &&thread) { m_data.push_back(thread); } + inline void add(int64_t affinity, int intensity = 1) { add(CpuThread(intensity, affinity)); } + inline void reserve(size_t capacity) { m_data.reserve(capacity); } + + rapidjson::Value toJSON(rapidjson::Document &doc) const; + +private: + std::vector m_data; +}; + + +} /* namespace xmrig */ + + +#endif /* XMRIG_CPUTHREADS_H */ diff --git a/src/backend/cpu/cpu.cmake b/src/backend/cpu/cpu.cmake index 2ae73db7..b6c8915b 100644 --- a/src/backend/cpu/cpu.cmake +++ b/src/backend/cpu/cpu.cmake @@ -4,6 +4,7 @@ set(HEADERS_BACKEND_CPU src/backend/cpu/CpuConfig.h src/backend/cpu/CpuLaunchData.cpp src/backend/cpu/CpuThread.h + src/backend/cpu/CpuThreads.h src/backend/cpu/CpuWorker.h src/backend/cpu/interfaces/ICpuInfo.h ) @@ -14,6 +15,7 @@ set(SOURCES_BACKEND_CPU src/backend/cpu/CpuConfig.cpp src/backend/cpu/CpuLaunchData.h src/backend/cpu/CpuThread.cpp + src/backend/cpu/CpuThreads.cpp src/backend/cpu/CpuWorker.cpp ) diff --git a/src/backend/cpu/interfaces/ICpuInfo.h b/src/backend/cpu/interfaces/ICpuInfo.h index daaa39c3..9bc3b11a 100644 --- a/src/backend/cpu/interfaces/ICpuInfo.h +++ b/src/backend/cpu/interfaces/ICpuInfo.h @@ -26,7 +26,7 @@ #define XMRIG_CPUINFO_H -#include "backend/cpu/CpuThread.h" +#include "backend/cpu/CpuThreads.h" #include "crypto/common/Assembly.h" #include "crypto/common/Algorithm.h" diff --git a/src/backend/cpu/platform/BasicCpuInfo.cpp b/src/backend/cpu/platform/BasicCpuInfo.cpp index 2b63edba..49d4b005 100644 --- a/src/backend/cpu/platform/BasicCpuInfo.cpp +++ b/src/backend/cpu/platform/BasicCpuInfo.cpp @@ -182,23 +182,25 @@ const char *xmrig::BasicCpuInfo::backend() const xmrig::CpuThreads xmrig::BasicCpuInfo::threads(const Algorithm &algorithm) const { - if (threads() == 1) { - return CpuThreads(1); + const size_t count = std::thread::hardware_concurrency(); + + if (count == 1) { + return 1; } # ifdef XMRIG_ALGO_CN_GPU if (algorithm == Algorithm::CN_GPU) { - return CpuThreads(threads()); + return count; } # endif if (algorithm.family() == Algorithm::CN_LITE || algorithm.family() == Algorithm::CN_PICO) { - return CpuThreads(threads()); + return count; } if (algorithm.family() == Algorithm::CN_HEAVY) { - return CpuThreads(std::max(threads() / 4, 1)); + return std::max(count / 4, 1); } - return CpuThreads(std::max(threads() / 2, 1)); + return std::max(count / 2, 1); } diff --git a/src/backend/cpu/platform/HwlocCpuInfo.cpp b/src/backend/cpu/platform/HwlocCpuInfo.cpp index b2aa47d7..8a9c75d3 100644 --- a/src/backend/cpu/platform/HwlocCpuInfo.cpp +++ b/src/backend/cpu/platform/HwlocCpuInfo.cpp @@ -222,7 +222,7 @@ xmrig::CpuThreads xmrig::HwlocCpuInfo::threads(const Algorithm &algorithm) const processTopLevelCache(cache, algorithm, threads); } - if (threads.empty()) { + if (threads.isEmpty()) { LOG_WARN("hwloc auto configuration for algorithm \"%s\" failed.", algorithm.shortName()); return BasicCpuInfo::threads(algorithm); @@ -286,7 +286,7 @@ void xmrig::HwlocCpuInfo::processTopLevelCache(hwloc_obj_t cache, const Algorith for (hwloc_obj_t core : cores) { const std::vector units = findByType(core, HWLOC_OBJ_PU); for (hwloc_obj_t pu : units) { - threads.push_back(CpuThread(1, pu->os_index)); + threads.add(pu->os_index); } } @@ -307,7 +307,7 @@ void xmrig::HwlocCpuInfo::processTopLevelCache(hwloc_obj_t cache, const Algorith PUs--; allocated_pu = true; - threads.push_back(CpuThread(1, units[pu_id]->os_index)); + threads.add(units[pu_id]->os_index); if (cacheHashes == 0) { break; From 42dc914eecb8c4b17efda3a6203bb06d723bf767 Mon Sep 17 00:00:00 2001 From: XMRig Date: Wed, 7 Aug 2019 18:12:39 +0700 Subject: [PATCH 14/26] Added alternative object format for CPU threads. --- doc/CPU.md | 6 ++ src/backend/common/Threads.cpp | 2 +- src/backend/cpu/CpuThreads.cpp | 90 +++++++++++++++++++++++++++-- src/backend/cpu/CpuThreads.h | 7 +++ src/core/config/ConfigTransform.cpp | 47 ++------------- 5 files changed, 106 insertions(+), 46 deletions(-) diff --git a/doc/CPU.md b/doc/CPU.md index f5d07745..15c3bf79 100644 --- a/doc/CPU.md +++ b/doc/CPU.md @@ -9,6 +9,7 @@ Example below demonstrate all primary ideas of flexible profiles configuration: * `"rx/wow"` Exact match to algorithm `rx/wow`, defined 4 threads without CPU affinity. * `"cn"` Default failback profile for all `cn/*` algorithms, defined 2 threads with CPU affinity, another failback profiles is `cn-lite`, `cn-heavy` and `rx`. * `"cn-lite"` Default failback profile for all `cn-lite/*` algorithms, defined 2 double threads with CPU affinity. +* `"cn-pico"` Alternative short object format, since 2.99.5. * `"custom-profile"` Custom user defined profile. * `"*"` Failback profile for all unhandled by other profiles algorithms. * `"cn/r"` Exact match, alias to profile `custom-profile`. @@ -34,6 +35,11 @@ Example below demonstrate all primary ideas of flexible profiles configuration: "affinity": 2 } ], + "cn-pico": { + "intensity": 2, + "threads": 8, + "affinity": -1 + }, "custom-profile": [0, 2], "*": [-1], "cn/r": "custom-profile", diff --git a/src/backend/common/Threads.cpp b/src/backend/common/Threads.cpp index 323352d1..17fc2951 100644 --- a/src/backend/common/Threads.cpp +++ b/src/backend/common/Threads.cpp @@ -55,7 +55,7 @@ size_t xmrig::Threads::read(const rapidjson::Value &value) using namespace rapidjson; for (auto &member : value.GetObject()) { - if (member.value.IsArray()) { + if (member.value.IsArray() || member.value.IsObject()) { T threads(member.value); if (!threads.isEmpty()) { diff --git a/src/backend/cpu/CpuThreads.cpp b/src/backend/cpu/CpuThreads.cpp index f877c9d3..2e8b9e1f 100644 --- a/src/backend/cpu/CpuThreads.cpp +++ b/src/backend/cpu/CpuThreads.cpp @@ -23,10 +23,66 @@ */ +#include + + #include "backend/cpu/CpuThreads.h" +#include "base/io/json/Json.h" #include "rapidjson/document.h" +namespace xmrig { + + +static const char *kAffinity = "affinity"; +static const char *kIntensity = "intensity"; +static const char *kThreads = "threads"; + + +static inline int64_t getAffinityMask(const rapidjson::Value &value) +{ + if (value.IsInt64()) { + return value.GetInt64(); + } + + if (value.IsString()) { + const char *arg = value.GetString(); + const char *p = strstr(arg, "0x"); + + return p ? strtoll(p, nullptr, 16) : strtoll(arg, nullptr, 10); + } + + return -1L; +} + + +static inline int64_t getAffinity(uint64_t index, int64_t affinity) +{ + if (affinity == -1L) { + return -1L; + } + + size_t idx = 0; + + for (size_t i = 0; i < 64; i++) { + if (!(static_cast(affinity) & (1ULL << i))) { + continue; + } + + if (idx == index) { + return static_cast(i); + } + + idx++; + } + + return -1L; +} + + +} + + xmrig::CpuThreads::CpuThreads(const rapidjson::Value &value) { if (value.IsArray()) { @@ -37,6 +93,20 @@ xmrig::CpuThreads::CpuThreads(const rapidjson::Value &value) } } } + else if (value.IsObject()) { + int intensity = Json::getInt(value, kIntensity, 1); + const size_t threads = std::min(Json::getUint(value, kThreads), 1024); + m_affinity = getAffinityMask(Json::getValue(value, kAffinity)); + m_format = ObjectFormat; + + if (intensity < 1 || intensity > 5) { + intensity = 1; + } + + for (size_t i = 0; i < threads; ++i) { + add(getAffinity(i, m_affinity), intensity); + } + } } @@ -45,10 +115,22 @@ rapidjson::Value xmrig::CpuThreads::toJSON(rapidjson::Document &doc) const using namespace rapidjson; auto &allocator = doc.GetAllocator(); - Value array(kArrayType); - for (const CpuThread &thread : m_data) { - array.PushBack(thread.toJSON(doc), allocator); + Value out; + + if (m_format == ArrayFormat) { + out.SetArray(); + + for (const CpuThread &thread : m_data) { + out.PushBack(thread.toJSON(doc), allocator); + } + } + else { + out.SetObject(); + + out.AddMember(StringRef(kIntensity), m_data.empty() ? 1 : m_data.front().intensity(), allocator); + out.AddMember(StringRef(kThreads), static_cast(m_data.size()), allocator); + out.AddMember(StringRef(kAffinity), m_affinity, allocator); } - return array; + return out; } diff --git a/src/backend/cpu/CpuThreads.h b/src/backend/cpu/CpuThreads.h index 3951c202..9d03e78b 100644 --- a/src/backend/cpu/CpuThreads.h +++ b/src/backend/cpu/CpuThreads.h @@ -53,6 +53,13 @@ public: rapidjson::Value toJSON(rapidjson::Document &doc) const; private: + enum Format { + ArrayFormat, + ObjectFormat + }; + + Format m_format = ArrayFormat; + int64_t m_affinity = -1; std::vector m_data; }; diff --git a/src/core/config/ConfigTransform.cpp b/src/core/config/ConfigTransform.cpp index ed315fb4..622855af 100644 --- a/src/core/config/ConfigTransform.cpp +++ b/src/core/config/ConfigTransform.cpp @@ -36,6 +36,7 @@ static const char *kAffinity = "affinity"; static const char *kAsterisk = "*"; static const char *kCpu = "cpu"; static const char *kIntensity = "intensity"; +static const char *kThreads = "threads"; #ifdef XMRIG_ALGO_RANDOMX static const char *kRandomX = "randomx"; @@ -79,30 +80,6 @@ static inline bool isHwAes(uint64_t av) } -static inline int64_t affinity(uint64_t index, int64_t affinity) -{ - if (affinity == -1L) { - return -1L; - } - - size_t idx = 0; - - for (size_t i = 0; i < 64; i++) { - if (!(static_cast(affinity) & (1ULL << i))) { - continue; - } - - if (idx == index) { - return static_cast(i); - } - - idx++; - } - - return -1L; -} - - } @@ -123,24 +100,12 @@ void xmrig::ConfigTransform::finalize(rapidjson::Document &doc) doc.AddMember(StringRef(kCpu), Value(kObjectType), allocator); } - Value threads(kArrayType); + Value profile(kObjectType); + profile.AddMember(StringRef(kIntensity), m_intensity, allocator); + profile.AddMember(StringRef(kThreads), m_threads, allocator); + profile.AddMember(StringRef(kAffinity), m_affinity, allocator); - if (m_intensity > 1) { - for (uint64_t i = 0; i < m_threads; ++i) { - Value thread(kObjectType); - thread.AddMember(StringRef(kIntensity), m_intensity, allocator); - thread.AddMember(StringRef(kAffinity), affinity(i, m_affinity), allocator); - - threads.PushBack(thread, doc.GetAllocator()); - } - } - else { - for (uint64_t i = 0; i < m_threads; ++i) { - threads.PushBack(affinity(i, m_affinity), doc.GetAllocator()); - } - } - - doc[kCpu].AddMember(StringRef(kAsterisk), threads, doc.GetAllocator()); + doc[kCpu].AddMember(StringRef(kAsterisk), profile, doc.GetAllocator()); } } From fd9039928b1fb144eb2f0d7b4a6fb93d6ba97685 Mon Sep 17 00:00:00 2001 From: XMRig Date: Wed, 7 Aug 2019 18:51:32 +0700 Subject: [PATCH 15/26] Added maxIntensity method to Algorithm. --- src/backend/common/Workers.cpp | 28 ++++------------------------ src/crypto/common/Algorithm.cpp | 18 ++++++++++++++++++ src/crypto/common/Algorithm.h | 1 + 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/backend/common/Workers.cpp b/src/backend/common/Workers.cpp index 6a369c1b..036bc8b6 100644 --- a/src/backend/common/Workers.cpp +++ b/src/backend/common/Workers.cpp @@ -163,36 +163,16 @@ void xmrig::Workers::onReady(void *arg) namespace xmrig { -#if defined (XMRIG_ALGO_RANDOMX) || defined (XMRIG_ALGO_CN_GPU) -static void printIntensityWarning(Thread *handle) -{ - LOG_WARN("CPU thread %zu warning: \"intensity %d not supported for %s algorithm\".", handle->index(), handle->config().intensity, handle->config().algorithm.shortName()); -} -#endif - - template<> xmrig::IWorker *xmrig::Workers::create(Thread *handle) { const int intensity = handle->config().intensity; -# if defined (XMRIG_ALGO_RANDOMX) || defined (XMRIG_ALGO_CN_GPU) - if (intensity > 1) { -# ifdef XMRIG_ALGO_RANDOMX - if (handle->config().algorithm.family() == Algorithm::RANDOM_X) { - printIntensityWarning(handle); +# if defined(XMRIG_ALGO_RANDOMX) || defined(XMRIG_ALGO_CN_GPU) + if (intensity > handle->config().algorithm.maxIntensity()) { + LOG_WARN("CPU thread %zu warning: \"intensity %d not supported for %s algorithm\".", handle->index(), handle->config().intensity, handle->config().algorithm.shortName()); - return new CpuWorker<1>(handle->index(), handle->config()); - } -# endif - -# ifdef XMRIG_ALGO_CN_GPU - if (handle->config().algorithm == Algorithm::CN_GPU) { - printIntensityWarning(handle); - - return new CpuWorker<1>(handle->index(), handle->config()); - } -# endif + return new CpuWorker<1>(handle->index(), handle->config()); } # endif diff --git a/src/crypto/common/Algorithm.cpp b/src/crypto/common/Algorithm.cpp index c7990052..f0670b26 100644 --- a/src/crypto/common/Algorithm.cpp +++ b/src/crypto/common/Algorithm.cpp @@ -120,6 +120,24 @@ static AlgoName const algorithm_names[] = { } /* namespace xmrig */ +int xmrig::Algorithm::maxIntensity() const +{ +# ifdef XMRIG_ALGO_RANDOMX + if (family() == RANDOM_X) { + return 1; + } +# endif + +# ifdef XMRIG_ALGO_CN_GPU + if (m_id == CN_GPU) { + return 1; + } +# endif + + return 5; +} + + rapidjson::Value xmrig::Algorithm::toJSON() const { using namespace rapidjson; diff --git a/src/crypto/common/Algorithm.h b/src/crypto/common/Algorithm.h index b30a946b..0b817bde 100644 --- a/src/crypto/common/Algorithm.h +++ b/src/crypto/common/Algorithm.h @@ -102,6 +102,7 @@ public: inline bool operator==(const Algorithm &other) const { return isEqual(other); } inline operator Algorithm::Id() const { return m_id; } + int maxIntensity() const; rapidjson::Value toJSON() const; size_t memory() const; From 97192f224d79a301c68b3b03818f502acd4cb1fd Mon Sep 17 00:00:00 2001 From: XMRig Date: Wed, 7 Aug 2019 21:47:42 +0700 Subject: [PATCH 16/26] Changed CPU threads format. --- doc/CPU.md | 50 +++++++++++++++++------ src/backend/cpu/CpuConfig.cpp | 17 ++++---- src/backend/cpu/CpuThread.cpp | 39 ++++++------------ src/backend/cpu/CpuThread.h | 7 ++-- src/backend/cpu/CpuThreads.h | 2 +- src/backend/cpu/platform/HwlocCpuInfo.cpp | 5 ++- 6 files changed, 68 insertions(+), 52 deletions(-) diff --git a/doc/CPU.md b/doc/CPU.md index 15c3bf79..756d3a0c 100644 --- a/doc/CPU.md +++ b/doc/CPU.md @@ -1,5 +1,7 @@ # CPU backend +**Information in this document actual to version 2.99.5+** + All CPU related settings contains in one `cpu` object in config file, CPU backend allow specify multiple profiles and allow switch between them without restrictions by pool request or config change. Default auto-configuration create reasonable minimum of profiles which cover all supported algorithms. ### Example @@ -9,7 +11,7 @@ Example below demonstrate all primary ideas of flexible profiles configuration: * `"rx/wow"` Exact match to algorithm `rx/wow`, defined 4 threads without CPU affinity. * `"cn"` Default failback profile for all `cn/*` algorithms, defined 2 threads with CPU affinity, another failback profiles is `cn-lite`, `cn-heavy` and `rx`. * `"cn-lite"` Default failback profile for all `cn-lite/*` algorithms, defined 2 double threads with CPU affinity. -* `"cn-pico"` Alternative short object format, since 2.99.5. +* `"cn-pico"` Alternative short object format. * `"custom-profile"` Custom user defined profile. * `"*"` Failback profile for all unhandled by other profiles algorithms. * `"cn/r"` Exact match, alias to profile `custom-profile`. @@ -24,16 +26,13 @@ Example below demonstrate all primary ideas of flexible profiles configuration: "priority": null, "asm": true, "rx/wow": [-1, -1, -1, -1], - "cn": [0, 2], + "cn": [ + [1, 0], + [1, 2] + ], "cn-lite": [ - { - "intensity": 2, - "affinity": 0 - }, - { - "intensity": 2, - "affinity": 2 - } + [2, 0], + [2, 2] ], "cn-pico": { "intensity": 2, @@ -48,8 +47,35 @@ Example below demonstrate all primary ideas of flexible profiles configuration: } ``` -### Intensity -This option was known as `low_power_mode`, possible values is range from 1 to 5, for convinient if value 1 used, possible omit this option and specify CPU thread config by only one number: CPU affinity, instead of object. +## Threads definition +Threads can be defined in 3 formats. + +#### Array format +```json +[ + [1, 0], + [1, 2], + [1, -1], + [2, -1] +] +``` +Each line represent one thread, first element is intensity, this option was known as `low_power_mode`, possible values is range from 1 to 5, second element is CPU affinity, special value `-1` means no affinity. + +#### Short array format +```json +[-1, -1, -1, -1] +``` +Each number represent one thread and means CPU affinity, this is default format for algorithm with maximum intensity 1, currently it all RandomX variants and cryptonight-gpu. + +#### Short object format +```json +{ + "intensity": 2, + "threads": 8, + "affinity": -1 +} +``` +Internal format, but can be user defined. ## Shared options diff --git a/src/backend/cpu/CpuConfig.cpp b/src/backend/cpu/CpuConfig.cpp index 883ef506..5905b7b4 100644 --- a/src/backend/cpu/CpuConfig.cpp +++ b/src/backend/cpu/CpuConfig.cpp @@ -147,31 +147,32 @@ void xmrig::CpuConfig::read(const rapidjson::Value &value) void xmrig::CpuConfig::generate() { - m_shouldSave = true; + m_shouldSave = true; + ICpuInfo *cpu = Cpu::info(); m_threads.disable(Algorithm::CN_0); - m_threads.move(kCn, Cpu::info()->threads(Algorithm::CN_0)); + m_threads.move(kCn, cpu->threads(Algorithm::CN_0)); # ifdef XMRIG_ALGO_CN_GPU - m_threads.move(kCnGPU, Cpu::info()->threads(Algorithm::CN_GPU)); + m_threads.move(kCnGPU, cpu->threads(Algorithm::CN_GPU)); # endif # ifdef XMRIG_ALGO_CN_LITE m_threads.disable(Algorithm::CN_LITE_0); - m_threads.move(kCnLite, Cpu::info()->threads(Algorithm::CN_LITE_1)); + m_threads.move(kCnLite, cpu->threads(Algorithm::CN_LITE_1)); # endif # ifdef XMRIG_ALGO_CN_HEAVY - m_threads.move(kCnHeavy, Cpu::info()->threads(Algorithm::CN_HEAVY_0)); + m_threads.move(kCnHeavy, cpu->threads(Algorithm::CN_HEAVY_0)); # endif # ifdef XMRIG_ALGO_CN_PICO - m_threads.move(kCnPico, Cpu::info()->threads(Algorithm::CN_PICO_0)); + m_threads.move(kCnPico, cpu->threads(Algorithm::CN_PICO_0)); # endif # ifdef XMRIG_ALGO_RANDOMX - m_threads.move(kRx, Cpu::info()->threads(Algorithm::RX_0)); - m_threads.move(kRxWOW, Cpu::info()->threads(Algorithm::RX_WOW)); + m_threads.move(kRx, cpu->threads(Algorithm::RX_0)); + m_threads.move(kRxWOW, cpu->threads(Algorithm::RX_WOW)); # endif } diff --git a/src/backend/cpu/CpuThread.cpp b/src/backend/cpu/CpuThread.cpp index e7132cfa..7d7a9e85 100644 --- a/src/backend/cpu/CpuThread.cpp +++ b/src/backend/cpu/CpuThread.cpp @@ -28,25 +28,14 @@ #include "rapidjson/document.h" -namespace xmrig { - - -static const char *kAffinity = "affinity"; -static const char *kIntensity = "intensity"; - - -} - - - xmrig::CpuThread::CpuThread(const rapidjson::Value &value) { - if (value.IsObject()) { - m_intensity = Json::getInt(value, kIntensity, -1); - m_affinity = Json::getInt(value, kAffinity, -1); + if (value.IsArray() && value.Size() >= 2) { + m_intensity = value[0].GetInt(); + m_affinity = value[1].GetInt(); } else if (value.IsInt()) { - m_intensity = 1; + m_intensity = -1; m_affinity = value.GetInt(); } } @@ -55,17 +44,15 @@ xmrig::CpuThread::CpuThread(const rapidjson::Value &value) rapidjson::Value xmrig::CpuThread::toJSON(rapidjson::Document &doc) const { using namespace rapidjson; - - if (intensity() > 1) { - auto &allocator = doc.GetAllocator(); - - Value obj(kObjectType); - - obj.AddMember(StringRef(kIntensity), m_intensity, allocator); - obj.AddMember(StringRef(kAffinity), m_affinity, allocator); - - return obj; + if (m_intensity == -1) { + return Value(m_affinity); } - return Value(m_affinity); + auto &allocator = doc.GetAllocator(); + + Value out(kArrayType); + out.PushBack(m_intensity, allocator); + out.PushBack(m_affinity, allocator); + + return out; } diff --git a/src/backend/cpu/CpuThread.h b/src/backend/cpu/CpuThread.h index 7c7ce4be..691ee114 100644 --- a/src/backend/cpu/CpuThread.h +++ b/src/backend/cpu/CpuThread.h @@ -35,13 +35,14 @@ namespace xmrig { class CpuThread { public: - inline constexpr CpuThread(int intensity = 1, int64_t affinity = -1) : m_intensity(intensity), m_affinity(affinity) {} + inline constexpr CpuThread() {} + inline constexpr CpuThread(int64_t affinity, int intensity) : m_intensity(intensity), m_affinity(affinity) {} CpuThread(const rapidjson::Value &value); inline bool isEqual(const CpuThread &other) const { return other.m_affinity == m_affinity && other.m_intensity == m_intensity; } - inline bool isValid() const { return m_intensity >= 1 && m_intensity <= 5; } - inline int intensity() const { return m_intensity; } + inline bool isValid() const { return m_intensity == -1 || (m_intensity >= 1 && m_intensity <= 5); } + inline int intensity() const { return m_intensity == -1 ? 1 : m_intensity; } inline int64_t affinity() const { return m_affinity; } inline bool operator!=(const CpuThread &other) const { return !isEqual(other); } diff --git a/src/backend/cpu/CpuThreads.h b/src/backend/cpu/CpuThreads.h index 9d03e78b..461cc1f2 100644 --- a/src/backend/cpu/CpuThreads.h +++ b/src/backend/cpu/CpuThreads.h @@ -47,7 +47,7 @@ public: inline const std::vector &data() const { return m_data; } inline size_t count() const { return m_data.size(); } inline void add(CpuThread &&thread) { m_data.push_back(thread); } - inline void add(int64_t affinity, int intensity = 1) { add(CpuThread(intensity, affinity)); } + inline void add(int64_t affinity, int intensity) { add(CpuThread(affinity, intensity)); } inline void reserve(size_t capacity) { m_data.reserve(capacity); } rapidjson::Value toJSON(rapidjson::Document &doc) const; diff --git a/src/backend/cpu/platform/HwlocCpuInfo.cpp b/src/backend/cpu/platform/HwlocCpuInfo.cpp index 8a9c75d3..c441457e 100644 --- a/src/backend/cpu/platform/HwlocCpuInfo.cpp +++ b/src/backend/cpu/platform/HwlocCpuInfo.cpp @@ -250,6 +250,7 @@ void xmrig::HwlocCpuInfo::processTopLevelCache(hwloc_obj_t cache, const Algorith int L2_associativity = 0; size_t extra = 0; const size_t scratchpad = algorithm.memory(); + int intensity = algorithm.maxIntensity() == 1 ? -1 : 1; if (cache->attr->cache.depth == 3 && isCacheExclusive(cache)) { for (size_t i = 0; i < cache->arity; ++i) { @@ -286,7 +287,7 @@ void xmrig::HwlocCpuInfo::processTopLevelCache(hwloc_obj_t cache, const Algorith for (hwloc_obj_t core : cores) { const std::vector units = findByType(core, HWLOC_OBJ_PU); for (hwloc_obj_t pu : units) { - threads.add(pu->os_index); + threads.add(pu->os_index, intensity); } } @@ -307,7 +308,7 @@ void xmrig::HwlocCpuInfo::processTopLevelCache(hwloc_obj_t cache, const Algorith PUs--; allocated_pu = true; - threads.add(units[pu_id]->os_index); + threads.add(units[pu_id]->os_index, intensity); if (cacheHashes == 0) { break; From 9a842a593b2d7a0148a6bdfb77e584c3af2064b4 Mon Sep 17 00:00:00 2001 From: XMRig Date: Thu, 8 Aug 2019 00:21:01 +0700 Subject: [PATCH 17/26] Use intensity=2 for cn-pico. --- .../AMD_Ryzen_7_3700X_windows_2_0_4.xml | 104 ++++++++++++++++++ src/backend/cpu/platform/HwlocCpuInfo.cpp | 6 + 2 files changed, 110 insertions(+) create mode 100644 doc/topology/AMD_Ryzen_7_3700X_windows_2_0_4.xml diff --git a/doc/topology/AMD_Ryzen_7_3700X_windows_2_0_4.xml b/doc/topology/AMD_Ryzen_7_3700X_windows_2_0_4.xml new file mode 100644 index 00000000..1f2d0ee4 --- /dev/null +++ b/doc/topology/AMD_Ryzen_7_3700X_windows_2_0_4.xml @@ -0,0 +1,104 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/cpu/platform/HwlocCpuInfo.cpp b/src/backend/cpu/platform/HwlocCpuInfo.cpp index c441457e..d88b4207 100644 --- a/src/backend/cpu/platform/HwlocCpuInfo.cpp +++ b/src/backend/cpu/platform/HwlocCpuInfo.cpp @@ -277,6 +277,12 @@ void xmrig::HwlocCpuInfo::processTopLevelCache(hwloc_obj_t cache, const Algorith size_t cacheHashes = ((L3 + extra) + (scratchpad / 2)) / scratchpad; +# ifdef XMRIG_ALGO_CN_PICO + if (algorithm == Algorithm::CN_PICO_0 && (cacheHashes / PUs) >= 2) { + intensity = 2; + } +# endif + # ifdef XMRIG_ALGO_CN_GPU if (algorithm == Algorithm::CN_GPU) { cacheHashes = PUs; From 5896b27bf3852177739fefe10e35c8d6a9ca68ac Mon Sep 17 00:00:00 2001 From: XMRig Date: Thu, 8 Aug 2019 14:03:52 +0700 Subject: [PATCH 18/26] Added L2 information to Algorithm. --- ...ntel_Xeon_Gold_6146_x2_UMA_linux_2_0_4.xml | 246 ++++++++++++++++++ src/backend/cpu/CpuBackend.cpp | 6 +- src/backend/cpu/CpuLaunchData.cpp | 2 +- src/backend/cpu/CpuWorker.cpp | 6 +- src/backend/cpu/platform/HwlocCpuInfo.cpp | 2 +- src/crypto/common/Algorithm.cpp | 22 +- src/crypto/common/Algorithm.h | 3 +- 7 files changed, 277 insertions(+), 10 deletions(-) create mode 100644 doc/topology/Intel_Xeon_Gold_6146_x2_UMA_linux_2_0_4.xml diff --git a/doc/topology/Intel_Xeon_Gold_6146_x2_UMA_linux_2_0_4.xml b/doc/topology/Intel_Xeon_Gold_6146_x2_UMA_linux_2_0_4.xml new file mode 100644 index 00000000..fe94194c --- /dev/null +++ b/doc/topology/Intel_Xeon_Gold_6146_x2_UMA_linux_2_0_4.xml @@ -0,0 +1,246 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/cpu/CpuBackend.cpp b/src/backend/cpu/CpuBackend.cpp index 7f208c38..e03046dd 100644 --- a/src/backend/cpu/CpuBackend.cpp +++ b/src/backend/cpu/CpuBackend.cpp @@ -99,13 +99,13 @@ public: tag, profileName.data(), threads.size(), - algo.memory() / 1024 + algo.l3() / 1024 ); workers.stop(); status.reset(); - status.memory = algo.memory(); + status.memory = algo.l3(); status.threads = threads.size(); for (const CpuLaunchData &data : threads) { @@ -317,7 +317,7 @@ rapidjson::Value xmrig::CpuBackend::toJSON(rapidjson::Document &doc) const hugepages.PushBack(pages[1], allocator); out.AddMember("hugepages", hugepages, allocator); - out.AddMember("memory", static_cast(d_ptr->algo.isValid() ? (ways * d_ptr->algo.memory()) : 0), allocator); + out.AddMember("memory", static_cast(d_ptr->algo.isValid() ? (ways * d_ptr->algo.l3()) : 0), allocator); if (d_ptr->threads.empty() || !hashrate()) { return out; diff --git a/src/backend/cpu/CpuLaunchData.cpp b/src/backend/cpu/CpuLaunchData.cpp index 6fa458aa..a01f22a6 100644 --- a/src/backend/cpu/CpuLaunchData.cpp +++ b/src/backend/cpu/CpuLaunchData.cpp @@ -43,7 +43,7 @@ xmrig::CpuLaunchData::CpuLaunchData(const Miner *miner, const Algorithm &algorit bool xmrig::CpuLaunchData::isEqual(const CpuLaunchData &other) const { - return (algorithm.memory() == other.algorithm.memory() + return (algorithm.l3() == other.algorithm.l3() && assembly == other.assembly && hugePages == other.hugePages && hwAES == other.hwAES diff --git a/src/backend/cpu/CpuWorker.cpp b/src/backend/cpu/CpuWorker.cpp index 000d7061..14ef1797 100644 --- a/src/backend/cpu/CpuWorker.cpp +++ b/src/backend/cpu/CpuWorker.cpp @@ -62,7 +62,7 @@ xmrig::CpuWorker::CpuWorker(size_t index, const CpuLaunchData &data) : m_miner(data.miner), m_ctx() { - m_memory = new VirtualMemory(m_algorithm.memory() * N, data.hugePages); + m_memory = new VirtualMemory(m_algorithm.l3() * N, data.hugePages); } @@ -184,7 +184,7 @@ void xmrig::CpuWorker::start() const Job &job = m_job.currentJob(); - if (job.algorithm().memory() != m_algorithm.memory()) { + if (job.algorithm().l3() != m_algorithm.l3()) { break; } @@ -283,7 +283,7 @@ template void xmrig::CpuWorker::allocateCnCtx() { if (m_ctx[0] == nullptr) { - CnCtx::create(m_ctx, m_memory->scratchpad(), m_algorithm.memory(), N); + CnCtx::create(m_ctx, m_memory->scratchpad(), m_algorithm.l3(), N); } } diff --git a/src/backend/cpu/platform/HwlocCpuInfo.cpp b/src/backend/cpu/platform/HwlocCpuInfo.cpp index d88b4207..047c97e8 100644 --- a/src/backend/cpu/platform/HwlocCpuInfo.cpp +++ b/src/backend/cpu/platform/HwlocCpuInfo.cpp @@ -249,7 +249,7 @@ void xmrig::HwlocCpuInfo::processTopLevelCache(hwloc_obj_t cache, const Algorith size_t L2 = 0; int L2_associativity = 0; size_t extra = 0; - const size_t scratchpad = algorithm.memory(); + const size_t scratchpad = algorithm.l3(); int intensity = algorithm.maxIntensity() == 1 ? -1 : 1; if (cache->attr->cache.depth == 3 && isCacheExclusive(cache)) { diff --git a/src/crypto/common/Algorithm.cpp b/src/crypto/common/Algorithm.cpp index f0670b26..30a7ad9c 100644 --- a/src/crypto/common/Algorithm.cpp +++ b/src/crypto/common/Algorithm.cpp @@ -146,7 +146,27 @@ rapidjson::Value xmrig::Algorithm::toJSON() const } -size_t xmrig::Algorithm::memory() const +size_t xmrig::Algorithm::l2() const +{ +# ifdef XMRIG_ALGO_RANDOMX + switch (m_id) { + case RX_0: + case RX_LOKI: + return 0x40000; + + case RX_WOW: + return 0x20000; + + default: + break; + } +# endif + + return 0; +} + + +size_t xmrig::Algorithm::l3() const { const Family f = family(); assert(f != UNKNOWN); diff --git a/src/crypto/common/Algorithm.h b/src/crypto/common/Algorithm.h index 0b817bde..92fcc61e 100644 --- a/src/crypto/common/Algorithm.h +++ b/src/crypto/common/Algorithm.h @@ -104,7 +104,8 @@ public: int maxIntensity() const; rapidjson::Value toJSON() const; - size_t memory() const; + size_t l2() const; + size_t l3() const; static Family family(Id id); static Id parse(const char *name); From b82181b9c59e9fd746eb5c4720b04b7abfad71a8 Mon Sep 17 00:00:00 2001 From: XMRig Date: Thu, 8 Aug 2019 21:03:29 +0700 Subject: [PATCH 19/26] #1103 Added L2 checks for RandomX auto configuration. --- src/backend/cpu/platform/HwlocCpuInfo.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/backend/cpu/platform/HwlocCpuInfo.cpp b/src/backend/cpu/platform/HwlocCpuInfo.cpp index 047c97e8..491305ec 100644 --- a/src/backend/cpu/platform/HwlocCpuInfo.cpp +++ b/src/backend/cpu/platform/HwlocCpuInfo.cpp @@ -289,6 +289,12 @@ void xmrig::HwlocCpuInfo::processTopLevelCache(hwloc_obj_t cache, const Algorith } # endif +# ifdef XMRIG_ALGO_RANDOMX + if (extra == 0 && algorithm.l2() > 0) { + cacheHashes = std::min(std::max(L2 / algorithm.l2(), cores.size()), cacheHashes); + } +# endif + if (cacheHashes >= PUs) { for (hwloc_obj_t core : cores) { const std::vector units = findByType(core, HWLOC_OBJ_PU); From ce370bf7212b054955f0deb6352ed590fcf4a1cb Mon Sep 17 00:00:00 2001 From: xmrig Date: Thu, 8 Aug 2019 22:28:27 +0700 Subject: [PATCH 20/26] Update CHANGELOG.md --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87cd1d4a..f2e3b081 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,10 @@ # v2.99.5-beta - [#1066](https://github.com/xmrig/xmrig/issues/1066#issuecomment-518080529) Fixed crash and added error message if pool not ready for RandomX. - [#1092](https://github.com/xmrig/xmrig/issues/1092) Fixed crash if wrong CPU affinity used. -- JSON arrays in config and API now more compact, single line if possible. +- [#1103](https://github.com/xmrig/xmrig/issues/1103) Improved auto configuration for RandomX for CPUs where L2 cache is limiting factor. +- [#1105](https://github.com/xmrig/xmrig/issues/1105) Improved auto configuration for `cn-pico` algorithm. +- Added alternative short format for CPU threads. +- Changed format for CPU threads with intensity above 1. # v2.99.4-beta - [#1062](https://github.com/xmrig/xmrig/issues/1062) Fixed 32 bit support. **32 bit is slow and deprecated**. From 647cbef43cdc9da428cb8e942461f30ce7ea87ea Mon Sep 17 00:00:00 2001 From: XMRig Date: Fri, 9 Aug 2019 01:00:21 +0700 Subject: [PATCH 21/26] uv_mutex replaced to std::mutex. --- src/backend/cpu/CpuBackend.cpp | 19 ++++------ src/crypto/common/Nonce.cpp | 14 +++----- src/net/JobResults.cpp | 65 +++++++++++++++------------------- 3 files changed, 39 insertions(+), 59 deletions(-) diff --git a/src/backend/cpu/CpuBackend.cpp b/src/backend/cpu/CpuBackend.cpp index e03046dd..863fa075 100644 --- a/src/backend/cpu/CpuBackend.cpp +++ b/src/backend/cpu/CpuBackend.cpp @@ -23,7 +23,7 @@ */ -#include +#include #include "backend/common/Hashrate.h" @@ -83,13 +83,6 @@ public: inline CpuBackendPrivate(Controller *controller) : controller(controller) { - uv_mutex_init(&mutex); - } - - - inline ~CpuBackendPrivate() - { - uv_mutex_destroy(&mutex); } @@ -119,9 +112,9 @@ public: Algorithm algo; Controller *controller; LaunchStatus status; + std::mutex mutex; std::vector threads; String profileName; - uv_mutex_t mutex; Workers workers; }; @@ -233,7 +226,7 @@ void xmrig::CpuBackend::setJob(const Job &job) void xmrig::CpuBackend::start(IWorker *worker) { - uv_mutex_lock(&d_ptr->mutex); + d_ptr->mutex.lock(); const auto pages = worker->memory()->hugePages(); @@ -254,7 +247,7 @@ void xmrig::CpuBackend::start(IWorker *worker) ); } - uv_mutex_unlock(&d_ptr->mutex); + d_ptr->mutex.unlock(); worker->start(); } @@ -299,10 +292,10 @@ rapidjson::Value xmrig::CpuBackend::toJSON(rapidjson::Document &doc) const out.AddMember("asm", false, allocator); # endif - uv_mutex_lock(&d_ptr->mutex); + d_ptr->mutex.lock(); uint64_t pages[2] = { d_ptr->status.hugePages, d_ptr->status.pages }; const size_t ways = d_ptr->status.ways; - uv_mutex_unlock(&d_ptr->mutex); + d_ptr->mutex.unlock(); # ifdef XMRIG_ALGO_RANDOMX if (d_ptr->algo.family() == Algorithm::RANDOM_X) { diff --git a/src/crypto/common/Nonce.cpp b/src/crypto/common/Nonce.cpp index 45c7001a..151819e0 100644 --- a/src/crypto/common/Nonce.cpp +++ b/src/crypto/common/Nonce.cpp @@ -23,7 +23,7 @@ */ -#include +#include #include "crypto/common/Nonce.h" @@ -37,7 +37,7 @@ std::atomic Nonce::m_sequence[Nonce::MAX]; uint32_t Nonce::m_nonces[2] = { 0, 0 }; -static uv_mutex_t mutex; +static std::mutex mutex; static Nonce nonce; @@ -51,8 +51,6 @@ xmrig::Nonce::Nonce() for (int i = 0; i < MAX; ++i) { m_sequence[i] = 1; } - - uv_mutex_init(&mutex); } @@ -60,7 +58,7 @@ uint32_t xmrig::Nonce::next(uint8_t index, uint32_t nonce, uint32_t reserveCount { uint32_t next; - uv_mutex_lock(&mutex); + std::lock_guard lock(mutex); if (nicehash) { next = (nonce & 0xFF000000) | m_nonces[index]; @@ -71,20 +69,16 @@ uint32_t xmrig::Nonce::next(uint8_t index, uint32_t nonce, uint32_t reserveCount m_nonces[index] += reserveCount; - uv_mutex_unlock(&mutex); - return next; } void xmrig::Nonce::reset(uint8_t index) { - uv_mutex_lock(&mutex); + std::lock_guard lock(mutex); m_nonces[index] = 0; touch(); - - uv_mutex_unlock(&mutex); } diff --git a/src/net/JobResults.cpp b/src/net/JobResults.cpp index bf0b5e86..40c2e50b 100644 --- a/src/net/JobResults.cpp +++ b/src/net/JobResults.cpp @@ -25,6 +25,7 @@ #include #include +#include #include @@ -40,75 +41,65 @@ namespace xmrig { class JobResultsPrivate { public: - inline JobResultsPrivate() + inline JobResultsPrivate(IJobResultListener *listener) : + listener(listener) { - uv_mutex_init(&m_mutex); + async = new uv_async_t; + async->data = this; - m_async = new uv_async_t; - m_async->data = this; - - uv_async_init(uv_default_loop(), m_async, JobResultsPrivate::onResult); + uv_async_init(uv_default_loop(), async, JobResultsPrivate::onResult); } inline ~JobResultsPrivate() { - Handle::close(m_async); - - uv_mutex_destroy(&m_mutex); - } - - - void setListener(IJobResultListener *listener) - { - m_listener = listener; + Handle::close(async); } void submit(const JobResult &result) { - uv_mutex_lock(&m_mutex); - m_queue.push_back(result); - uv_mutex_unlock(&m_mutex); + mutex.lock(); + queue.push_back(result); + mutex.unlock(); - uv_async_send(m_async); + uv_async_send(async); } private: - static void onResult(uv_async_t *handle) - { - static_cast(handle->data)->submit(); - } + static void onResult(uv_async_t *handle) { static_cast(handle->data)->submit(); } inline void submit() { std::list results; - uv_mutex_lock(&m_mutex); - while (!m_queue.empty()) { - results.push_back(std::move(m_queue.front())); - m_queue.pop_front(); + mutex.lock(); + + while (!queue.empty()) { + results.push_back(std::move(queue.front())); + queue.pop_front(); } - uv_mutex_unlock(&m_mutex); + + mutex.unlock(); for (auto result : results) { - m_listener->onJobResult(result); + listener->onJobResult(result); } results.clear(); } - IJobResultListener *m_listener = nullptr; - std::list m_queue; - uv_async_t *m_async; - uv_mutex_t m_mutex; + IJobResultListener *listener; + std::list queue; + std::mutex mutex; + uv_async_t *async; }; -static JobResultsPrivate *handler = new JobResultsPrivate(); +static JobResultsPrivate *handler = nullptr; } // namespace xmrig @@ -117,14 +108,16 @@ static JobResultsPrivate *handler = new JobResultsPrivate(); void xmrig::JobResults::setListener(IJobResultListener *listener) { - assert(handler != nullptr && listener != nullptr); + assert(handler == nullptr); - handler->setListener(listener); + handler = new JobResultsPrivate(listener); } void xmrig::JobResults::stop() { + assert(handler != nullptr); + delete handler; handler = nullptr; From 7a6790d0f6ae62a6e18e6ce637d8fd0bd7bf04cb Mon Sep 17 00:00:00 2001 From: XMRig Date: Fri, 9 Aug 2019 01:50:31 +0700 Subject: [PATCH 22/26] #1106 Fixed hugepages field in API. --- src/backend/cpu/CpuBackend.cpp | 46 +++++++++++++++++++++++----------- src/backend/cpu/CpuBackend.h | 5 ++++ src/core/Miner.cpp | 18 +++++++++++-- 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/src/backend/cpu/CpuBackend.cpp b/src/backend/cpu/CpuBackend.cpp index 863fa075..24600270 100644 --- a/src/backend/cpu/CpuBackend.cpp +++ b/src/backend/cpu/CpuBackend.cpp @@ -109,6 +109,14 @@ public: } + size_t ways() + { + std::lock_guard lock(mutex); + + return status.ways; + } + + Algorithm algo; Controller *controller; LaunchStatus status; @@ -135,6 +143,25 @@ xmrig::CpuBackend::~CpuBackend() } +std::pair xmrig::CpuBackend::hugePages() const +{ + std::pair pages(0, 0); + +# ifdef XMRIG_ALGO_RANDOMX + if (d_ptr->algo.family() == Algorithm::RANDOM_X) { + pages = Rx::hugePages(); + } +# endif + + std::lock_guard lock(d_ptr->mutex); + + pages.first += d_ptr->status.hugePages; + pages.second += d_ptr->status.pages; + + return pages; +} + + bool xmrig::CpuBackend::isEnabled() const { return d_ptr->controller->config()->cpu().isEnabled(); @@ -292,25 +319,14 @@ rapidjson::Value xmrig::CpuBackend::toJSON(rapidjson::Document &doc) const out.AddMember("asm", false, allocator); # endif - d_ptr->mutex.lock(); - uint64_t pages[2] = { d_ptr->status.hugePages, d_ptr->status.pages }; - const size_t ways = d_ptr->status.ways; - d_ptr->mutex.unlock(); - -# ifdef XMRIG_ALGO_RANDOMX - if (d_ptr->algo.family() == Algorithm::RANDOM_X) { - const auto rxPages = Rx::hugePages(); - pages[0] += rxPages.first; - pages[1] += rxPages.second; - } -# endif + const auto pages = hugePages(); rapidjson::Value hugepages(rapidjson::kArrayType); - hugepages.PushBack(pages[0], allocator); - hugepages.PushBack(pages[1], allocator); + hugepages.PushBack(pages.first, allocator); + hugepages.PushBack(pages.second, allocator); out.AddMember("hugepages", hugepages, allocator); - out.AddMember("memory", static_cast(d_ptr->algo.isValid() ? (ways * d_ptr->algo.l3()) : 0), allocator); + out.AddMember("memory", static_cast(d_ptr->algo.isValid() ? (d_ptr->ways() * d_ptr->algo.l3()) : 0), allocator); if (d_ptr->threads.empty() || !hashrate()) { return out; diff --git a/src/backend/cpu/CpuBackend.h b/src/backend/cpu/CpuBackend.h index 9e71c247..bedef5ca 100644 --- a/src/backend/cpu/CpuBackend.h +++ b/src/backend/cpu/CpuBackend.h @@ -26,6 +26,9 @@ #define XMRIG_CPUBACKEND_H +#include + + #include "backend/common/interfaces/IBackend.h" @@ -43,6 +46,8 @@ public: CpuBackend(Controller *controller); ~CpuBackend() override; + std::pair hugePages() const; + protected: bool isEnabled() const override; bool isEnabled(const Algorithm &algorithm) const override; diff --git a/src/core/Miner.cpp b/src/core/Miner.cpp index a1c65ed2..4406ce52 100644 --- a/src/core/Miner.cpp +++ b/src/core/Miner.cpp @@ -145,10 +145,24 @@ public: reply.AddMember("ua", StringRef(Platform::userAgent()), allocator); reply.AddMember("cpu", Cpu::toJSON(doc), allocator); - if (version == 1) { - reply.AddMember("hugepages", false, allocator); + Value hugepages; + + if (!backends.empty() && backends.front()->type() == "cpu") { + const auto pages = static_cast(backends.front())->hugePages(); + if (version > 1) { + hugepages.SetArray(); + hugepages.PushBack(pages.first, allocator); + hugepages.PushBack(pages.second, allocator); + } + else { + hugepages = pages.first == pages.second; + } + } + else { + hugepages = false; } + reply.AddMember("hugepages", hugepages, allocator); reply.AddMember("donate_level", controller->config()->pools().donateLevel(), allocator); Value algo(kArrayType); From be251a2ec19f7aeb78c33a3fea28b72f621e9207 Mon Sep 17 00:00:00 2001 From: xmrig Date: Fri, 9 Aug 2019 02:09:20 +0700 Subject: [PATCH 23/26] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2e3b081..f2ad4ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - [#1092](https://github.com/xmrig/xmrig/issues/1092) Fixed crash if wrong CPU affinity used. - [#1103](https://github.com/xmrig/xmrig/issues/1103) Improved auto configuration for RandomX for CPUs where L2 cache is limiting factor. - [#1105](https://github.com/xmrig/xmrig/issues/1105) Improved auto configuration for `cn-pico` algorithm. +- [#1106](https://github.com/xmrig/xmrig/issues/1106) Fixed `hugepages` field in summary API. - Added alternative short format for CPU threads. - Changed format for CPU threads with intensity above 1. From 4583d979db793eabdaac399dd7fcae3f2cbd7c5a Mon Sep 17 00:00:00 2001 From: XMRig Date: Fri, 9 Aug 2019 12:51:27 +0700 Subject: [PATCH 24/26] Fixed auto configuration without hwloc. --- src/backend/cpu/CpuThreads.cpp | 10 +++++++ src/backend/cpu/CpuThreads.h | 1 + src/backend/cpu/platform/AdvancedCpuInfo.cpp | 14 +++++++--- src/backend/cpu/platform/BasicCpuInfo.cpp | 28 +++++++++++++++++--- 4 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/backend/cpu/CpuThreads.cpp b/src/backend/cpu/CpuThreads.cpp index 2e8b9e1f..07e8ca33 100644 --- a/src/backend/cpu/CpuThreads.cpp +++ b/src/backend/cpu/CpuThreads.cpp @@ -110,6 +110,16 @@ xmrig::CpuThreads::CpuThreads(const rapidjson::Value &value) } +xmrig::CpuThreads::CpuThreads(size_t count, int intensity) +{ + m_data.reserve(count); + + for (size_t i = 0; i < count; ++i) { + add(-1, intensity); + } +} + + rapidjson::Value xmrig::CpuThreads::toJSON(rapidjson::Document &doc) const { using namespace rapidjson; diff --git a/src/backend/cpu/CpuThreads.h b/src/backend/cpu/CpuThreads.h index 461cc1f2..f8ad6430 100644 --- a/src/backend/cpu/CpuThreads.h +++ b/src/backend/cpu/CpuThreads.h @@ -42,6 +42,7 @@ public: inline CpuThreads(size_t count) : m_data(count) {} CpuThreads(const rapidjson::Value &value); + CpuThreads(size_t count, int intensity); inline bool isEmpty() const { return m_data.empty(); } inline const std::vector &data() const { return m_data; } diff --git a/src/backend/cpu/platform/AdvancedCpuInfo.cpp b/src/backend/cpu/platform/AdvancedCpuInfo.cpp index f8871d3c..de8ff272 100644 --- a/src/backend/cpu/platform/AdvancedCpuInfo.cpp +++ b/src/backend/cpu/platform/AdvancedCpuInfo.cpp @@ -112,7 +112,7 @@ xmrig::AdvancedCpuInfo::AdvancedCpuInfo() : xmrig::CpuThreads xmrig::AdvancedCpuInfo::threads(const Algorithm &algorithm) const { if (threads() == 1) { - return CpuThreads(1); + return 1; } # ifdef XMRIG_ALGO_CN_GPU @@ -132,7 +132,7 @@ xmrig::CpuThreads xmrig::AdvancedCpuInfo::threads(const Algorithm &algorithm) co } if (cache) { - const size_t memory = algorithm.memory(); + const size_t memory = algorithm.l3(); assert(memory > 0); count = cache / memory; @@ -145,5 +145,13 @@ xmrig::CpuThreads xmrig::AdvancedCpuInfo::threads(const Algorithm &algorithm) co count = threads() / 2; } - return CpuThreads(std::max(std::min(count, threads()), 1)); + int intensity = algorithm.maxIntensity() == 1 ? -1 : 1; + +# ifdef XMRIG_ALGO_CN_PICO + if (algorithm == Algorithm::CN_PICO_0 && (count / cores()) >= 2) { + intensity = 2; + } +# endif + + return CpuThreads(std::max(std::min(count, threads()), 1), intensity); } diff --git a/src/backend/cpu/platform/BasicCpuInfo.cpp b/src/backend/cpu/platform/BasicCpuInfo.cpp index 49d4b005..2f3177d2 100644 --- a/src/backend/cpu/platform/BasicCpuInfo.cpp +++ b/src/backend/cpu/platform/BasicCpuInfo.cpp @@ -194,13 +194,33 @@ xmrig::CpuThreads xmrig::BasicCpuInfo::threads(const Algorithm &algorithm) const } # endif - if (algorithm.family() == Algorithm::CN_LITE || algorithm.family() == Algorithm::CN_PICO) { - return count; +# ifdef XMRIG_ALGO_CN_LITE + if (algorithm.family() == Algorithm::CN_LITE) { + return CpuThreads(count, 1); } +# endif +# ifdef XMRIG_ALGO_CN_PICO + if (algorithm.family() == Algorithm::CN_PICO) { + return CpuThreads(count, 2); + } +# endif + +# ifdef XMRIG_ALGO_CN_HEAVY if (algorithm.family() == Algorithm::CN_HEAVY) { - return std::max(count / 4, 1); + return CpuThreads(std::max(count / 4, 1), 1); } +# endif - return std::max(count / 2, 1); +# ifdef XMRIG_ALGO_RANDOMX + if (algorithm.family() == Algorithm::RANDOM_X) { + if (algorithm == Algorithm::RX_WOW) { + return count; + } + + return std::max(count / 2, 1); + } +# endif + + return CpuThreads(std::max(count / 2, 1), 1); } From 5cb3ef90680e95fe96108d6a4d5e964b5ee87301 Mon Sep 17 00:00:00 2001 From: XMRig Date: Fri, 9 Aug 2019 14:26:00 +0700 Subject: [PATCH 25/26] Name for reference RandomX configuration changed to rx/text to avoid potential conflicts in future. --- CHANGELOG.md | 1 + doc/ALGORITHMS.md | 1 + src/base/net/stratum/Job.cpp | 9 ++++++++- src/crypto/common/Algorithm.cpp | 2 ++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2ad4ed1..b67934bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - [#1106](https://github.com/xmrig/xmrig/issues/1106) Fixed `hugepages` field in summary API. - Added alternative short format for CPU threads. - Changed format for CPU threads with intensity above 1. +- Name for reference RandomX configuration changed to `rx/text` to avoid potential conflicts in future. # v2.99.4-beta - [#1062](https://github.com/xmrig/xmrig/issues/1062) Fixed 32 bit support. **32 bit is slow and deprecated**. diff --git a/doc/ALGORITHMS.md b/doc/ALGORITHMS.md index 07f92271..076eb2ff 100644 --- a/doc/ALGORITHMS.md +++ b/doc/ALGORITHMS.md @@ -44,6 +44,7 @@ Since version 3 mining [algorithm](#algorithm-names) should specified for each p | `cn-heavy/xhv` | 4 MB | CryptoNight-Heavy (modified, TUBE only). | | `cn-heavy/tube` | 4 MB | CryptoNight-Heavy (modified, Haven Protocol only). | | `cn-pico` | 256 KB | TurtleCoin (TRTL) | +| `rx/test` | 2 MB | RandomX (reference configuration). | | `rx/0` | 2 MB | RandomX (reference configuration), reserved for future use. | | `rx/wow` | 1 MB | RandomWOW (Wownero). | | `rx/loki` | 2 MB | RandomXL (Loki). | diff --git a/src/base/net/stratum/Job.cpp b/src/base/net/stratum/Job.cpp index 512b686e..04bd82d8 100644 --- a/src/base/net/stratum/Job.cpp +++ b/src/base/net/stratum/Job.cpp @@ -174,6 +174,13 @@ void xmrig::Job::copy(const Job &other) m_target = other.m_target; m_index = other.m_index; - memcpy(m_blob, other.m_blob, sizeof (m_blob)); + memcpy(m_blob, other.m_blob, sizeof(m_blob)); memcpy(m_seedHash, other.m_seedHash, sizeof(m_seedHash)); + +# ifdef XMRIG_PROXY_PROJECT + m_rawSeedHash = other.m_rawSeedHash; + + memcpy(m_rawBlob, other.m_rawBlob, sizeof(m_rawBlob)); + memcpy(m_rawTarget, other.m_rawTarget, sizeof(m_rawTarget)); +# endif } diff --git a/src/crypto/common/Algorithm.cpp b/src/crypto/common/Algorithm.cpp index 30a7ad9c..db6cb234 100644 --- a/src/crypto/common/Algorithm.cpp +++ b/src/crypto/common/Algorithm.cpp @@ -107,6 +107,8 @@ static AlgoName const algorithm_names[] = { { "cryptonight_turtle", "cn_turtle", Algorithm::CN_PICO_0 }, # endif # ifdef XMRIG_ALGO_RANDOMX + { "randomx/test", "rx/test", Algorithm::RX_0 }, + { "randomx/0", "rx/0", Algorithm::RX_0 }, { "randomx/0", "rx/0", Algorithm::RX_0 }, { "RandomX", "rx", Algorithm::RX_0 }, { "randomx/wow", "rx/wow", Algorithm::RX_WOW }, From feda02bf5021fde035d58ee062f15cb76e0dbc83 Mon Sep 17 00:00:00 2001 From: XMRig Date: Fri, 9 Aug 2019 15:36:34 +0700 Subject: [PATCH 26/26] Disable HW AES for RandomX on ARM. --- src/crypto/rx/RxVm.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/crypto/rx/RxVm.cpp b/src/crypto/rx/RxVm.cpp index 6426443a..275f9558 100644 --- a/src/crypto/rx/RxVm.cpp +++ b/src/crypto/rx/RxVm.cpp @@ -33,9 +33,11 @@ xmrig::RxVm::RxVm(RxDataset *dataset, uint8_t *scratchpad, bool softAes) { +# ifndef XMRIG_ARM if (!softAes) { m_flags |= RANDOMX_FLAG_HARD_AES; } +# endif if (dataset->get()) { m_flags |= RANDOMX_FLAG_FULL_MEM;