forked from dlang/cdcdb
Обновлена схема БД - добавлены таблицы users, processes, files. Добавлен параметр Context с идентификацией пользователей и процесса. label сменен на file.
This commit is contained in:
parent
5bb4d65c92
commit
49ee7a4053
GPG key ID:
C8D8BE544A27C511
7 changed files with 289 additions and 168 deletions
|
|
@ -17,7 +17,7 @@ enum SnapshotStatus : ubyte
|
|||
|
||||
struct DBSnapshot {
|
||||
long id;
|
||||
string label;
|
||||
string file;
|
||||
ubyte[32] sha256;
|
||||
string description;
|
||||
DateTime createdUtc;
|
||||
|
|
@ -28,6 +28,11 @@ struct DBSnapshot {
|
|||
long maskS;
|
||||
long maskL;
|
||||
SnapshotStatus status;
|
||||
long uid;
|
||||
long ruid;
|
||||
string uidName;
|
||||
string ruidName;
|
||||
string process;
|
||||
}
|
||||
|
||||
struct DBSnapshotChunk
|
||||
|
|
@ -104,7 +109,8 @@ private:
|
|||
{
|
||||
SqliteResult queryResult = sql(
|
||||
q{
|
||||
WITH required(name) AS (VALUES ("snapshots"), ("blobs"), ("snapshot_chunks"))
|
||||
WITH required(name)
|
||||
AS (VALUES ("snapshots"), ("blobs"), ("snapshot_chunks"), ("users"), ("processes"), ("files"))
|
||||
SELECT name AS missing_table
|
||||
FROM required
|
||||
WHERE NOT EXISTS (
|
||||
|
|
@ -122,11 +128,11 @@ private:
|
|||
missingTables ~= row["missing_table"].to!string;
|
||||
}
|
||||
|
||||
enforce(missingTables.length == 0 || missingTables.length == 3,
|
||||
enforce(missingTables.length == 0 || missingTables.length == 6,
|
||||
"Database is corrupted. Missing tables: " ~ missingTables.join(", ")
|
||||
);
|
||||
|
||||
if (missingTables.length == 3)
|
||||
if (missingTables.length == 6)
|
||||
{
|
||||
foreach (schemeQuery; _scheme)
|
||||
{
|
||||
|
|
@ -172,17 +178,21 @@ public:
|
|||
sql("ROLLBACK");
|
||||
}
|
||||
|
||||
bool isLast(string label, ubyte[] sha256) {
|
||||
bool isLast(string file, ubyte[] sha256) {
|
||||
auto queryResult = sql(
|
||||
q{
|
||||
SELECT COALESCE(
|
||||
(SELECT (label = ? AND sha256 = ?)
|
||||
FROM snapshots
|
||||
ORDER BY created_utc DESC
|
||||
LIMIT 1),
|
||||
(
|
||||
SELECT (s.sha256 = ?2)
|
||||
FROM snapshots s
|
||||
JOIN files f ON f.id = s.file
|
||||
WHERE f.name = ?1
|
||||
ORDER BY s.created_utc DESC
|
||||
LIMIT 1
|
||||
),
|
||||
0
|
||||
) AS is_last;
|
||||
}, label, sha256
|
||||
}, file, sha256
|
||||
);
|
||||
|
||||
if (!queryResult.empty())
|
||||
|
|
@ -195,23 +205,34 @@ public:
|
|||
auto queryResult = sql(
|
||||
q{
|
||||
INSERT INTO snapshots(
|
||||
label,
|
||||
file,
|
||||
sha256,
|
||||
description,
|
||||
source_length,
|
||||
uid,
|
||||
ruid,
|
||||
process,
|
||||
algo_min,
|
||||
algo_normal,
|
||||
algo_max,
|
||||
mask_s,
|
||||
mask_l,
|
||||
status
|
||||
) VALUES (?,?,?,?,?,?,?,?,?,?)
|
||||
)
|
||||
SELECT
|
||||
(SELECT id FROM files WHERE name = ?),
|
||||
?,?,?,?,?,
|
||||
(SELECT id FROM processes WHERE name = ?),
|
||||
?,?,?,?,?,?
|
||||
RETURNING id
|
||||
},
|
||||
snapshot.label,
|
||||
snapshot.file,
|
||||
snapshot.sha256[],
|
||||
snapshot.description.length ? snapshot.description : null,
|
||||
snapshot.sourceLength,
|
||||
snapshot.uid,
|
||||
snapshot.ruid,
|
||||
snapshot.process,
|
||||
snapshot.algoMin,
|
||||
snapshot.algoNormal,
|
||||
snapshot.algoMax,
|
||||
|
|
@ -247,6 +268,43 @@ public:
|
|||
return !queryResult.empty();
|
||||
}
|
||||
|
||||
bool addProcess(string name)
|
||||
{
|
||||
auto queryResult = sql(
|
||||
q{
|
||||
INSERT INTO processes (name) VALUES (?)
|
||||
ON CONFLICT(name) DO NOTHING
|
||||
}, name
|
||||
);
|
||||
|
||||
return !queryResult.empty();
|
||||
}
|
||||
|
||||
bool addFile(string name)
|
||||
{
|
||||
auto queryResult = sql(
|
||||
q{
|
||||
INSERT INTO files (name) VALUES (?)
|
||||
ON CONFLICT(name) DO NOTHING
|
||||
}, name
|
||||
);
|
||||
|
||||
return !queryResult.empty();
|
||||
}
|
||||
|
||||
bool addUser(long uid, string name)
|
||||
{
|
||||
auto queryResult = sql(
|
||||
q{
|
||||
INSERT INTO users (uid, name)
|
||||
VALUES (?, ?)
|
||||
ON CONFLICT(uid) DO NOTHING;
|
||||
}, uid, name
|
||||
);
|
||||
|
||||
return !queryResult.empty();
|
||||
}
|
||||
|
||||
bool addSnapshotChunk(DBSnapshotChunk snapshotChunk)
|
||||
{
|
||||
auto queryResult = sql(
|
||||
|
|
@ -268,9 +326,30 @@ public:
|
|||
{
|
||||
auto queryResult = sql(
|
||||
q{
|
||||
SELECT id, label, sha256, description, created_utc, source_length,
|
||||
algo_min, algo_normal, algo_max, mask_s, mask_l, status
|
||||
FROM snapshots WHERE id = ?
|
||||
SELECT
|
||||
s.id,
|
||||
f.name file,
|
||||
s.sha256,
|
||||
s.description,
|
||||
s.created_utc,
|
||||
s.source_length,
|
||||
s.uid,
|
||||
s.ruid,
|
||||
u.name uid_name,
|
||||
r.name ruid_name,
|
||||
p.name process,
|
||||
s.algo_min,
|
||||
s.algo_normal,
|
||||
s.algo_max,
|
||||
s.mask_s,
|
||||
s.mask_l,
|
||||
s.status
|
||||
FROM snapshots s
|
||||
JOIN processes p ON p.id = s.process
|
||||
JOIN users u ON u.uid = s.uid
|
||||
JOIN users r ON r.uid = s.ruid
|
||||
JOIN files f ON f.id = s.file
|
||||
WHERE s.id = ?;
|
||||
}, id
|
||||
);
|
||||
|
||||
|
|
@ -281,7 +360,7 @@ public:
|
|||
auto data = queryResult.front();
|
||||
|
||||
snapshot.id = data["id"].to!long;
|
||||
snapshot.label = data["label"].to!string;
|
||||
snapshot.file = data["file"].to!string;
|
||||
snapshot.sha256 = cast(ubyte[]) data["sha256"].dup;
|
||||
snapshot.description = data["description"].to!string;
|
||||
snapshot.createdUtc = toDateTime(data["created_utc"].to!string);
|
||||
|
|
@ -292,19 +371,45 @@ public:
|
|||
snapshot.maskS = data["mask_s"].to!long;
|
||||
snapshot.maskL = data["mask_l"].to!long;
|
||||
snapshot.status = cast(SnapshotStatus) data["status"].to!int;
|
||||
snapshot.uid = data["uid"].to!long;
|
||||
snapshot.ruid = data["ruid"].to!long;
|
||||
snapshot.uidName = data["uid_name"].to!string;
|
||||
snapshot.ruidName = data["ruid_name"].to!string;
|
||||
snapshot.process = data["process"].to!string;
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
DBSnapshot[] getSnapshots(string label)
|
||||
DBSnapshot[] getSnapshots(string file)
|
||||
{
|
||||
auto queryResult = sql(
|
||||
q{
|
||||
SELECT id, label, sha256, description, created_utc, source_length,
|
||||
algo_min, algo_normal, algo_max, mask_s, mask_l, status
|
||||
FROM snapshots WHERE (length(?) = 0 OR label = ?1);
|
||||
}, label
|
||||
SELECT
|
||||
s.id,
|
||||
f.name file,
|
||||
s.sha256,
|
||||
s.description,
|
||||
s.created_utc,
|
||||
s.source_length,
|
||||
s.uid,
|
||||
s.ruid,
|
||||
u.name uid_name,
|
||||
r.name ruid_name,
|
||||
p.name process,
|
||||
s.algo_min,
|
||||
s.algo_normal,
|
||||
s.algo_max,
|
||||
s.mask_s,
|
||||
s.mask_l,
|
||||
s.status
|
||||
FROM snapshots s
|
||||
JOIN processes p ON p.id = s.process
|
||||
JOIN users u ON u.uid = s.uid
|
||||
JOIN users r ON r.uid = s.ruid
|
||||
JOIN files f ON f.id = s.file AND (length(?) = 0 OR f.name = ?1)
|
||||
ORDER BY s.created_utc, s.id;
|
||||
}, file
|
||||
);
|
||||
|
||||
DBSnapshot[] snapshots;
|
||||
|
|
@ -314,7 +419,7 @@ public:
|
|||
DBSnapshot snapshot;
|
||||
|
||||
snapshot.id = row["id"].to!long;
|
||||
snapshot.label = row["label"].to!string;
|
||||
snapshot.file = row["file"].to!string;
|
||||
snapshot.sha256 = cast(ubyte[]) row["sha256"].dup;
|
||||
snapshot.description = row["description"].to!string;
|
||||
snapshot.createdUtc = toDateTime(row["created_utc"].to!string);
|
||||
|
|
@ -325,6 +430,11 @@ public:
|
|||
snapshot.maskS = row["mask_s"].to!long;
|
||||
snapshot.maskL = row["mask_l"].to!long;
|
||||
snapshot.status = cast(SnapshotStatus) row["status"].to!int;
|
||||
snapshot.uid = row["uid"].to!long;
|
||||
snapshot.ruid = row["ruid"].to!long;
|
||||
snapshot.uidName = row["uid_name"].to!string;
|
||||
snapshot.ruidName = row["ruid_name"].to!string;
|
||||
snapshot.process = row["process"].to!string;
|
||||
|
||||
snapshots ~= snapshot;
|
||||
}
|
||||
|
|
@ -375,13 +485,13 @@ public:
|
|||
return 0;
|
||||
}
|
||||
|
||||
long deleteSnapshot(string label) {
|
||||
long deleteSnapshots(string file) {
|
||||
auto queryResult = sql(
|
||||
q{
|
||||
DELETE FROM snapshots
|
||||
WHERE label = ?
|
||||
WHERE file = (SELECT id FROM files WHERE name = ?)
|
||||
RETURNING 1;
|
||||
}, label);
|
||||
}, file);
|
||||
|
||||
if (!queryResult.empty()) {
|
||||
return queryResult.length.to!long;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,52 @@
|
|||
auto _scheme = [
|
||||
q{
|
||||
-- ------------------------------------------------------------
|
||||
-- Таблица users
|
||||
-- ------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
-- Linux UID
|
||||
uid INTEGER PRIMARY KEY,
|
||||
-- текстовое представление пользователя
|
||||
name TEXT NOT NULL UNIQUE
|
||||
)
|
||||
},
|
||||
q{
|
||||
-- Индекс по имени пользователя
|
||||
CREATE INDEX IF NOT EXISTS idx_users_name
|
||||
ON users(name)
|
||||
},
|
||||
q{
|
||||
-- ------------------------------------------------------------
|
||||
-- Таблица processes
|
||||
-- ------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS processes (
|
||||
-- идентификатор процесса
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
-- имя процесса
|
||||
name TEXT NOT NULL UNIQUE
|
||||
)
|
||||
},
|
||||
q{
|
||||
-- Индекс по имени процесса
|
||||
CREATE INDEX IF NOT EXISTS idx_processes_name
|
||||
ON processes(name)
|
||||
},
|
||||
q{
|
||||
-- ------------------------------------------------------------
|
||||
-- Таблица files
|
||||
-- ------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
-- идентификатор процесса
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
-- имя процесса
|
||||
name TEXT NOT NULL UNIQUE
|
||||
)
|
||||
},
|
||||
q{
|
||||
-- Индекс по имени процесса
|
||||
CREATE INDEX IF NOT EXISTS idx_processes_name
|
||||
ON processes(name)
|
||||
},
|
||||
q{
|
||||
-- ------------------------------------------------------------
|
||||
-- Таблица snapshots
|
||||
|
|
@ -6,8 +54,8 @@ auto _scheme = [
|
|||
CREATE TABLE IF NOT EXISTS snapshots (
|
||||
-- идентификатор снимка
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
-- метка/название снимка
|
||||
label TEXT NOT NULL,
|
||||
-- Файл
|
||||
file INTEGER NOT NULL,
|
||||
-- SHA-256 всего файла (BLOB(32))
|
||||
sha256 BLOB NOT NULL CHECK (length(sha256) = 32),
|
||||
-- Комментарий/описание
|
||||
|
|
@ -16,6 +64,12 @@ auto _scheme = [
|
|||
created_utc TEXT NOT NULL DEFAULT (CURRENT_TIMESTAMP),
|
||||
-- длина исходного файла в байтах
|
||||
source_length INTEGER NOT NULL,
|
||||
-- UID пользователя (эффективный)
|
||||
uid INTEGER NOT NULL,
|
||||
-- RUID пользователя (реальный)
|
||||
ruid INTEGER NOT NULL,
|
||||
-- Процесс
|
||||
process INTEGER NOT NULL,
|
||||
-- FastCDC: минимальный размер чанка
|
||||
algo_min INTEGER NOT NULL,
|
||||
-- FastCDC: целевой размер чанка
|
||||
|
|
@ -28,7 +82,24 @@ auto _scheme = [
|
|||
mask_l INTEGER NOT NULL,
|
||||
-- 0=pending, 1=ready
|
||||
status INTEGER NOT NULL DEFAULT 0
|
||||
CHECK (status IN (0,1))
|
||||
CHECK (status IN (0,1)),
|
||||
-- Внешние ключи
|
||||
FOREIGN KEY (uid)
|
||||
REFERENCES users(uid)
|
||||
ON UPDATE RESTRICT
|
||||
ON DELETE RESTRICT,
|
||||
FOREIGN KEY (ruid)
|
||||
REFERENCES users(uid)
|
||||
ON UPDATE RESTRICT
|
||||
ON DELETE RESTRICT,
|
||||
FOREIGN KEY (process)
|
||||
REFERENCES processes(id)
|
||||
ON UPDATE RESTRICT
|
||||
ON DELETE RESTRICT
|
||||
FOREIGN KEY (file)
|
||||
REFERENCES files(id)
|
||||
ON UPDATE CASCADE
|
||||
ON DELETE CASCADE
|
||||
)
|
||||
},
|
||||
q{
|
||||
|
|
@ -88,9 +159,9 @@ auto _scheme = [
|
|||
)
|
||||
},
|
||||
q{
|
||||
-- Индекс для запросов вида: WHERE label=? AND sha256=?
|
||||
-- Индекс для запросов вида: WHERE file=? AND sha256=?
|
||||
CREATE INDEX IF NOT EXISTS idx_snapshots_path_sha
|
||||
ON snapshots(label, sha256)
|
||||
ON snapshots(file, sha256)
|
||||
},
|
||||
q{
|
||||
-- Индекс для обратного поиска использования blob по sha256
|
||||
|
|
@ -215,11 +286,14 @@ auto _scheme = [
|
|||
FOR EACH ROW
|
||||
WHEN
|
||||
NEW.id IS NOT OLD.id OR
|
||||
NEW.label IS NOT OLD.label OR
|
||||
NEW.file IS NOT OLD.file OR
|
||||
NEW.sha256 IS NOT OLD.sha256 OR
|
||||
NEW.description IS NOT OLD.description OR
|
||||
NEW.created_utc IS NOT OLD.created_utc OR
|
||||
NEW.source_length IS NOT OLD.source_length OR
|
||||
NEW.uid IS NOT OLD.uid OR
|
||||
NEW.ruid IS NOT OLD.ruid OR
|
||||
NEW.process IS NOT OLD.process OR
|
||||
NEW.algo_min IS NOT OLD.algo_min OR
|
||||
NEW.algo_normal IS NOT OLD.algo_normal OR
|
||||
NEW.algo_max IS NOT OLD.algo_max OR
|
||||
|
|
@ -249,5 +323,20 @@ auto _scheme = [
|
|||
BEGIN
|
||||
SELECT RAISE(ABORT, "blobs: разрешён UPDATE только полей last_seen_utc и refcount");
|
||||
END
|
||||
},
|
||||
q{
|
||||
-- ------------------------------------------------------------
|
||||
-- Удаление записи из files, если удалён последний snapshot
|
||||
-- ------------------------------------------------------------
|
||||
CREATE TRIGGER IF NOT EXISTS trg_snapshots_delete_file
|
||||
AFTER DELETE ON snapshots
|
||||
FOR EACH ROW
|
||||
BEGIN
|
||||
DELETE FROM files
|
||||
WHERE id = OLD.file
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM snapshots WHERE file = OLD.file
|
||||
);
|
||||
END;
|
||||
}
|
||||
];
|
||||
|
|
|
|||
|
|
@ -167,9 +167,9 @@ public:
|
|||
}
|
||||
|
||||
/// User-defined label.
|
||||
@property string label() const @safe
|
||||
@property string file() const @safe
|
||||
{
|
||||
return _snapshot.label;
|
||||
return _snapshot.file;
|
||||
}
|
||||
|
||||
/// Creation timestamp (UTC) from the database.
|
||||
|
|
|
|||
|
|
@ -6,6 +6,14 @@ import cdcdb.snapshot;
|
|||
|
||||
import zstd : compress, Level;
|
||||
|
||||
struct Context {
|
||||
long uid;
|
||||
long ruid;
|
||||
string uidName;
|
||||
string ruidName;
|
||||
string process;
|
||||
}
|
||||
|
||||
/**
|
||||
* High-level storage facade: splits data into CDC chunks, stores chunks/blobs
|
||||
* into SQLite via `DBLite`, links them into snapshots, and returns `Snapshot`
|
||||
|
|
@ -97,7 +105,7 @@ public:
|
|||
///
|
||||
/// Throws:
|
||||
/// Exception if `data` is empty or on database/storage errors
|
||||
Snapshot newSnapshot(string label, const(ubyte)[] data, string description = string.init)
|
||||
Snapshot newSnapshot(string file, const(ubyte)[] data, Context context, string description = string.init)
|
||||
{
|
||||
if (data.length == 0)
|
||||
{
|
||||
|
|
@ -109,21 +117,9 @@ public:
|
|||
ubyte[32] sha256 = digest!SHA256(data);
|
||||
|
||||
// If the last snapshot for the label matches current content
|
||||
if (_db.isLast(label, sha256))
|
||||
if (_db.isLast(file, sha256))
|
||||
return null;
|
||||
|
||||
DBSnapshot dbSnapshot;
|
||||
|
||||
dbSnapshot.label = label;
|
||||
dbSnapshot.sha256 = sha256;
|
||||
dbSnapshot.description = description;
|
||||
dbSnapshot.sourceLength = data.length;
|
||||
dbSnapshot.algoMin = _minSize;
|
||||
dbSnapshot.algoNormal = _normalSize;
|
||||
dbSnapshot.algoMax = _maxSize;
|
||||
dbSnapshot.maskS = _maskS;
|
||||
dbSnapshot.maskL = _maskL;
|
||||
|
||||
_db.beginImmediate();
|
||||
|
||||
bool ok;
|
||||
|
|
@ -138,6 +134,30 @@ public:
|
|||
_db.commit();
|
||||
}
|
||||
|
||||
_db.addUser(context.uid, context.uidName);
|
||||
if (context.uid != context.ruid) {
|
||||
_db.addUser(context.ruid, context.ruidName);
|
||||
}
|
||||
|
||||
_db.addFile(file);
|
||||
_db.addProcess(context.process);
|
||||
|
||||
DBSnapshot dbSnapshot;
|
||||
|
||||
dbSnapshot.file = file;
|
||||
dbSnapshot.sha256 = sha256;
|
||||
dbSnapshot.description = description;
|
||||
dbSnapshot.sourceLength = data.length;
|
||||
dbSnapshot.algoMin = _minSize;
|
||||
dbSnapshot.algoNormal = _normalSize;
|
||||
dbSnapshot.algoMax = _maxSize;
|
||||
dbSnapshot.maskS = _maskS;
|
||||
dbSnapshot.maskL = _maskL;
|
||||
|
||||
dbSnapshot.uid = context.uid;
|
||||
dbSnapshot.ruid = context.ruid;
|
||||
dbSnapshot.process = context.process;
|
||||
|
||||
auto idSnapshot = _db.addSnapshot(dbSnapshot);
|
||||
|
||||
DBSnapshotChunk dbSnapshotChunk;
|
||||
|
|
@ -193,8 +213,8 @@ public:
|
|||
/// label = snapshot label
|
||||
///
|
||||
/// Returns: number of deleted snapshots
|
||||
long removeSnapshots(string label) {
|
||||
return _db.deleteSnapshot(label);
|
||||
long removeSnapshots(string file) {
|
||||
return _db.deleteSnapshots(file);
|
||||
}
|
||||
|
||||
/// Delete a specific snapshot instance.
|
||||
|
|
@ -203,8 +223,8 @@ public:
|
|||
/// snapshot = `Snapshot` to remove
|
||||
///
|
||||
/// Returns: `true` on success, `false` otherwise
|
||||
bool removeSnapshots(Snapshot snapshot) {
|
||||
return removeSnapshots(snapshot.id);
|
||||
bool removeSnapshot(Snapshot snapshot) {
|
||||
return removeSnapshot(snapshot.id);
|
||||
}
|
||||
|
||||
/// Delete a snapshot by id.
|
||||
|
|
@ -213,7 +233,7 @@ public:
|
|||
/// idSnapshot = snapshot id
|
||||
///
|
||||
/// Returns: `true` if the row was deleted
|
||||
bool removeSnapshots(long idSnapshot) {
|
||||
bool removeSnapshot(long idSnapshot) {
|
||||
return _db.deleteSnapshot(idSnapshot) == idSnapshot;
|
||||
}
|
||||
|
||||
|
|
@ -233,10 +253,10 @@ public:
|
|||
/// label = filter by exact label; empty string returns all
|
||||
///
|
||||
/// Returns: array of `Snapshot` handles
|
||||
Snapshot[] getSnapshots(string label = string.init) {
|
||||
Snapshot[] getSnapshots(string file = string.init) {
|
||||
Snapshot[] snapshots;
|
||||
|
||||
foreach (snapshot; _db.getSnapshots(label)) {
|
||||
foreach (snapshot; _db.getSnapshots(file)) {
|
||||
snapshots ~= new Snapshot(_db, snapshot);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue