mirror of
https://github.com/revng/revng
synced 2026-06-21 14:07:57 +00:00
Giacomo Vercesi
a06b380ce0
Allow uploads to S3 to be executed in parallel, this should reduce the time it takes for a `save` operation to conclude.
80 lines
2.1 KiB
C++
80 lines
2.1 KiB
C++
#pragma once
|
|
|
|
//
|
|
// This file is distributed under the MIT License. See LICENSE.md for details.
|
|
//
|
|
|
|
#include "aws/core/auth/AWSCredentials.h"
|
|
#include "aws/s3/S3Client.h"
|
|
|
|
#include "llvm/ADT/StringMap.h"
|
|
#include "llvm/ADT/StringRef.h"
|
|
#include "llvm/Support/ThreadPool.h"
|
|
|
|
#include "revng/Storage/StorageClient.h"
|
|
|
|
namespace revng {
|
|
|
|
class AsyncUploadTask;
|
|
class S3WritableFile;
|
|
|
|
class S3StorageClient : public StorageClient {
|
|
private:
|
|
struct UploadResult {
|
|
Aws::S3::Model::PutObjectOutcome Outcome;
|
|
std::string Path;
|
|
std::string NewPath;
|
|
std::string Error;
|
|
};
|
|
|
|
private:
|
|
Aws::Auth::AWSCredentials Credentials;
|
|
Aws::S3::S3Client Client;
|
|
std::string Bucket;
|
|
std::string SubPath;
|
|
std::string RedactedURL;
|
|
llvm::ThreadPoolTaskGroup TaskGroup;
|
|
std::vector<std::shared_future<UploadResult>> PendingUploads;
|
|
llvm::StringMap<std::string> FilenameMap;
|
|
static constexpr auto IndexName = "index.yml";
|
|
|
|
public:
|
|
S3StorageClient(llvm::StringRef URL);
|
|
~S3StorageClient() override = default;
|
|
|
|
static llvm::Expected<std::unique_ptr<S3StorageClient>>
|
|
fromURL(llvm::StringRef URL);
|
|
|
|
static bool isS3URL(llvm::StringRef Input) {
|
|
return Input.starts_with("s3://") or Input.starts_with("s3s://");
|
|
}
|
|
|
|
llvm::Expected<PathType> type(llvm::StringRef Path) override;
|
|
llvm::Error createDirectory(llvm::StringRef Path) override;
|
|
llvm::Error remove(llvm::StringRef Path) override;
|
|
llvm::sys::path::Style getStyle() const override;
|
|
|
|
llvm::Error copy(llvm::StringRef Source,
|
|
llvm::StringRef Destination) override;
|
|
|
|
llvm::Expected<std::unique_ptr<ReadableFile>>
|
|
getReadableFile(llvm::StringRef Path) override;
|
|
|
|
llvm::Expected<std::unique_ptr<WritableFile>>
|
|
getWritableFile(llvm::StringRef Path, ContentEncoding Encoding) override;
|
|
|
|
llvm::Error commit() override;
|
|
|
|
// In S3StorageClient, the Credentials are in the format:
|
|
// '<username>:<password>'
|
|
llvm::Error setCredentials(llvm::StringRef Credentials) override;
|
|
|
|
private:
|
|
std::string dumpString() const override;
|
|
std::string resolvePath(llvm::StringRef Path);
|
|
friend class S3WritableFile;
|
|
friend class AsyncUploadTask;
|
|
};
|
|
|
|
} // namespace revng
|