59bcec2685
We were previously using the primary OnDiskGraphDB when validating the upstream OnDiskKeyValueDB, which is incorrect since the values being stored are direct offsets and therefore cannot be used across DBs without translating to a hash value first. rdar://170067863
258 lines
9.0 KiB
C++
258 lines
9.0 KiB
C++
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
///
|
|
/// \file This file implements the underlying ActionCache implementations.
|
|
///
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "BuiltinCAS.h"
|
|
#include "llvm/ADT/TrieRawHashMap.h"
|
|
#include "llvm/CAS/ActionCache.h"
|
|
#include "llvm/CAS/OnDiskCASLogger.h"
|
|
#include "llvm/CAS/OnDiskKeyValueDB.h"
|
|
#include "llvm/CAS/UnifiedOnDiskCache.h"
|
|
#include "llvm/Config/llvm-config.h"
|
|
#include "llvm/Support/BLAKE3.h"
|
|
#include "llvm/Support/Errc.h"
|
|
|
|
#define DEBUG_TYPE "cas-action-caches"
|
|
|
|
using namespace llvm;
|
|
using namespace llvm::cas;
|
|
|
|
namespace {
|
|
|
|
using HasherT = BLAKE3;
|
|
using HashType = decltype(HasherT::hash(std::declval<ArrayRef<uint8_t> &>()));
|
|
|
|
template <size_t Size> class CacheEntry {
|
|
public:
|
|
CacheEntry() = default;
|
|
CacheEntry(ArrayRef<uint8_t> Hash) { llvm::copy(Hash, Value.data()); }
|
|
CacheEntry(const CacheEntry &Entry) { llvm::copy(Entry.Value, Value.data()); }
|
|
ArrayRef<uint8_t> getValue() const { return Value; }
|
|
|
|
private:
|
|
std::array<uint8_t, Size> Value;
|
|
};
|
|
|
|
/// Builtin InMemory ActionCache that stores the mapping in memory.
|
|
class InMemoryActionCache final : public ActionCache {
|
|
public:
|
|
InMemoryActionCache()
|
|
: ActionCache(builtin::BuiltinCASContext::getDefaultContext()) {}
|
|
|
|
Error putImpl(ArrayRef<uint8_t> ActionKey, const CASID &Result,
|
|
bool CanBeDistributed) final;
|
|
Expected<std::optional<CASID>> getImpl(ArrayRef<uint8_t> ActionKey,
|
|
bool CanBeDistributed) const final;
|
|
|
|
Error validate() const final {
|
|
return createStringError("InMemoryActionCache doesn't support validate()");
|
|
}
|
|
|
|
private:
|
|
using DataT = CacheEntry<sizeof(HashType)>;
|
|
using InMemoryCacheT = ThreadSafeTrieRawHashMap<DataT, sizeof(HashType)>;
|
|
|
|
InMemoryCacheT Cache;
|
|
};
|
|
|
|
/// Builtin basic OnDiskActionCache that uses one underlying OnDiskKeyValueDB.
|
|
class OnDiskActionCache final : public ActionCache {
|
|
public:
|
|
Error putImpl(ArrayRef<uint8_t> ActionKey, const CASID &Result,
|
|
bool CanBeDistributed) final;
|
|
Expected<std::optional<CASID>> getImpl(ArrayRef<uint8_t> ActionKey,
|
|
bool CanBeDistributed) const final;
|
|
|
|
static Expected<std::unique_ptr<OnDiskActionCache>> create(StringRef Path);
|
|
|
|
Error validate() const final;
|
|
|
|
private:
|
|
static StringRef getHashName() { return "BLAKE3"; }
|
|
|
|
OnDiskActionCache(std::unique_ptr<ondisk::OnDiskKeyValueDB> DB);
|
|
|
|
std::unique_ptr<ondisk::OnDiskKeyValueDB> DB;
|
|
using DataT = CacheEntry<sizeof(HashType)>;
|
|
};
|
|
|
|
/// Builtin unified ActionCache that wraps around UnifiedOnDiskCache to provide
|
|
/// access to its ActionCache.
|
|
class UnifiedOnDiskActionCache final : public ActionCache {
|
|
public:
|
|
Error putImpl(ArrayRef<uint8_t> ActionKey, const CASID &Result,
|
|
bool CanBeDistributed) final;
|
|
Expected<std::optional<CASID>> getImpl(ArrayRef<uint8_t> ActionKey,
|
|
bool CanBeDistributed) const final;
|
|
|
|
UnifiedOnDiskActionCache(std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB);
|
|
|
|
Error validate() const final;
|
|
|
|
private:
|
|
std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB;
|
|
};
|
|
} // end namespace
|
|
|
|
static Error createResultCachePoisonedError(ArrayRef<uint8_t> KeyHash,
|
|
const CASContext &Context,
|
|
CASID Output,
|
|
ArrayRef<uint8_t> ExistingOutput) {
|
|
std::string Existing =
|
|
CASID::create(&Context, toStringRef(ExistingOutput)).toString();
|
|
SmallString<64> Key;
|
|
toHex(KeyHash, /*LowerCase=*/true, Key);
|
|
return createStringError(std::make_error_code(std::errc::invalid_argument),
|
|
"cache poisoned for '" + Key + "' (new='" +
|
|
Output.toString() + "' vs. existing '" +
|
|
Existing + "')");
|
|
}
|
|
|
|
Expected<std::optional<CASID>>
|
|
InMemoryActionCache::getImpl(ArrayRef<uint8_t> Key,
|
|
bool /*CanBeDistributed*/) const {
|
|
auto Result = Cache.find(Key);
|
|
if (!Result)
|
|
return std::nullopt;
|
|
return CASID::create(&getContext(), toStringRef(Result->Data.getValue()));
|
|
}
|
|
|
|
Error InMemoryActionCache::putImpl(ArrayRef<uint8_t> Key, const CASID &Result,
|
|
bool /*CanBeDistributed*/) {
|
|
DataT Expected(Result.getHash());
|
|
const InMemoryCacheT::value_type &Cached = *Cache.insertLazy(
|
|
Key, [&](auto ValueConstructor) { ValueConstructor.emplace(Expected); });
|
|
|
|
const DataT &Observed = Cached.Data;
|
|
if (Expected.getValue() == Observed.getValue())
|
|
return Error::success();
|
|
|
|
return createResultCachePoisonedError(Key, getContext(), Result,
|
|
Observed.getValue());
|
|
}
|
|
|
|
namespace llvm::cas {
|
|
|
|
std::unique_ptr<ActionCache> createInMemoryActionCache() {
|
|
return std::make_unique<InMemoryActionCache>();
|
|
}
|
|
|
|
} // namespace llvm::cas
|
|
|
|
OnDiskActionCache::OnDiskActionCache(
|
|
std::unique_ptr<ondisk::OnDiskKeyValueDB> DB)
|
|
: ActionCache(builtin::BuiltinCASContext::getDefaultContext()),
|
|
DB(std::move(DB)) {}
|
|
|
|
Expected<std::unique_ptr<OnDiskActionCache>>
|
|
OnDiskActionCache::create(StringRef AbsPath) {
|
|
std::shared_ptr<ondisk::OnDiskCASLogger> Logger;
|
|
#ifndef _WIN32
|
|
if (Error E =
|
|
ondisk::OnDiskCASLogger::openIfEnabled(AbsPath).moveInto(Logger))
|
|
return std::move(E);
|
|
#endif
|
|
std::unique_ptr<ondisk::OnDiskKeyValueDB> DB;
|
|
if (Error E = ondisk::OnDiskKeyValueDB::open(
|
|
AbsPath, getHashName(), sizeof(HashType), getHashName(),
|
|
sizeof(DataT), /*UnifiedCache=*/nullptr, std::move(Logger))
|
|
.moveInto(DB))
|
|
return std::move(E);
|
|
return std::unique_ptr<OnDiskActionCache>(
|
|
new OnDiskActionCache(std::move(DB)));
|
|
}
|
|
|
|
Expected<std::optional<CASID>>
|
|
OnDiskActionCache::getImpl(ArrayRef<uint8_t> Key,
|
|
bool /*CanBeDistributed*/) const {
|
|
std::optional<ArrayRef<char>> Val;
|
|
if (Error E = DB->get(Key).moveInto(Val))
|
|
return std::move(E);
|
|
if (!Val)
|
|
return std::nullopt;
|
|
return CASID::create(&getContext(), toStringRef(*Val));
|
|
}
|
|
|
|
Error OnDiskActionCache::putImpl(ArrayRef<uint8_t> Key, const CASID &Result,
|
|
bool /*CanBeDistributed*/) {
|
|
auto ResultHash = Result.getHash();
|
|
ArrayRef Expected((const char *)ResultHash.data(), ResultHash.size());
|
|
ArrayRef<char> Observed;
|
|
if (Error E = DB->put(Key, Expected).moveInto(Observed))
|
|
return E;
|
|
|
|
if (Expected == Observed)
|
|
return Error::success();
|
|
|
|
return createResultCachePoisonedError(
|
|
Key, getContext(), Result,
|
|
ArrayRef((const uint8_t *)Observed.data(), Observed.size()));
|
|
}
|
|
|
|
Error OnDiskActionCache::validate() const { return DB->validate(); }
|
|
|
|
UnifiedOnDiskActionCache::UnifiedOnDiskActionCache(
|
|
std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB)
|
|
: ActionCache(builtin::BuiltinCASContext::getDefaultContext()),
|
|
UniDB(std::move(UniDB)) {}
|
|
|
|
Expected<std::optional<CASID>>
|
|
UnifiedOnDiskActionCache::getImpl(ArrayRef<uint8_t> Key,
|
|
bool /*CanBeDistributed*/) const {
|
|
std::optional<ArrayRef<char>> Val;
|
|
if (Error E = UniDB->getKeyValueDB().get(Key).moveInto(Val))
|
|
return std::move(E);
|
|
if (!Val)
|
|
return std::nullopt;
|
|
auto ID = ondisk::UnifiedOnDiskCache::getObjectIDFromValue(*Val);
|
|
return CASID::create(&getContext(),
|
|
toStringRef(UniDB->getGraphDB().getDigest(ID)));
|
|
}
|
|
|
|
Error UnifiedOnDiskActionCache::putImpl(ArrayRef<uint8_t> Key,
|
|
const CASID &Result,
|
|
bool /*CanBeDistributed*/) {
|
|
auto Expected = UniDB->getGraphDB().getReference(Result.getHash());
|
|
if (LLVM_UNLIKELY(!Expected))
|
|
return Expected.takeError();
|
|
|
|
auto Value = ondisk::UnifiedOnDiskCache::getValueFromObjectID(*Expected);
|
|
std::optional<ArrayRef<char>> Observed;
|
|
if (Error E = UniDB->getKeyValueDB().put(Key, Value).moveInto(Observed))
|
|
return E;
|
|
|
|
auto ObservedID = ondisk::UnifiedOnDiskCache::getObjectIDFromValue(*Observed);
|
|
if (*Expected == ObservedID)
|
|
return Error::success();
|
|
|
|
return createResultCachePoisonedError(
|
|
Key, getContext(), Result, UniDB->getGraphDB().getDigest(ObservedID));
|
|
}
|
|
|
|
Error UnifiedOnDiskActionCache::validate() const {
|
|
return UniDB->validateActionCache();
|
|
}
|
|
|
|
Expected<std::unique_ptr<ActionCache>>
|
|
cas::createOnDiskActionCache(StringRef Path) {
|
|
#if LLVM_ENABLE_ONDISK_CAS
|
|
return OnDiskActionCache::create(Path);
|
|
#else
|
|
return createStringError(inconvertibleErrorCode(), "OnDiskCache is disabled");
|
|
#endif
|
|
}
|
|
|
|
std::unique_ptr<ActionCache>
|
|
cas::builtin::createActionCacheFromUnifiedOnDiskCache(
|
|
std::shared_ptr<ondisk::UnifiedOnDiskCache> UniDB) {
|
|
return std::make_unique<UnifiedOnDiskActionCache>(std::move(UniDB));
|
|
}
|