From 066643ec13bf7cfefc0532ec1dd2b77432ab8a12 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Tue, 15 Nov 2011 12:34:40 +1100 Subject: [PATCH 1/7] ldb: Add handy macros for reporting error inside ldb module Pair-Programmed-With: Andrew Tridgell Autobuild-User: Amitay Isaacs Autobuild-Date: Thu Nov 17 05:24:46 CET 2011 on sn-devel-104 (cherry picked from commit 3a0f7b89d551a739d886396d320309f8e7d04372) --- lib/ldb/include/ldb_module.h | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/lib/ldb/include/ldb_module.h b/lib/ldb/include/ldb_module.h index 2acd8f3..4ecddc4 100644 --- a/lib/ldb/include/ldb_module.h +++ b/lib/ldb/include/ldb_module.h @@ -85,10 +85,12 @@ void ldb_debug_add(struct ldb_context *ldb, const char *fmt, ...) PRINTF_ATTRIBU void ldb_debug_end(struct ldb_context *ldb, enum ldb_debug_level level); #define ldb_error(ldb, ecode, reason) ldb_error_at(ldb, ecode, reason, __FILE__, __LINE__) +#define ldb_module_error(module, ecode, reason) ldb_error_at(ldb_module_get_ctx(module), ecode, reason, __FILE__, __LINE__) #define ldb_oom(ldb) ldb_error(ldb, LDB_ERR_OPERATIONS_ERROR, "ldb out of memory") #define ldb_module_oom(module) ldb_oom(ldb_module_get_ctx(module)) #define ldb_operr(ldb) ldb_error(ldb, LDB_ERR_OPERATIONS_ERROR, "operations error") +#define ldb_module_operr(module) ldb_error(ldb_module_get_ctx(module), LDB_ERR_OPERATIONS_ERROR, "operations error") /* The following definitions come from lib/ldb/common/ldb.c */ -- 1.7.2.5 From fc964415207b01f4c989ff05a99b19625f297553 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Tue, 22 Nov 2011 16:51:04 +1100 Subject: [PATCH 2/7] s4-dsdb: Remove LDB_SEQ_HIGHEST_TIMESTAMP sequence number support This was a hack for LDAP backends to store a sequence number as a timestamp. It is still supported in standalone ldb tdb backend. Signed-off-by: Andrew Tridgell (cherry picked from commit c199b35dd46b2de21d89ed93edb9a815035717fc) --- source4/dsdb/samdb/ldb_modules/partition.c | 146 ++-------------------- source4/dsdb/samdb/ldb_modules/simple_ldap_map.c | 7 +- 2 files changed, 13 insertions(+), 140 deletions(-) diff --git a/source4/dsdb/samdb/ldb_modules/partition.c b/source4/dsdb/samdb/ldb_modules/partition.c index 92918c7..9ab53ae 100644 --- a/source4/dsdb/samdb/ldb_modules/partition.c +++ b/source4/dsdb/samdb/ldb_modules/partition.c @@ -1002,13 +1002,12 @@ int partition_primary_sequence_number(struct ldb_module *module, TALLOC_CTX *mem seqr = talloc_get_type(res->extended->data, struct ldb_seqnum_result); if (seqr->flags & LDB_SEQ_TIMESTAMP_SEQUENCE) { - ret = LDB_ERR_OPERATIONS_ERROR; - ldb_set_errstring(ldb_module_get_ctx(module), "Primary backend in partitions module returned a timestamp based seq number (must return a normal number)"); talloc_free(res); - return ret; - } else { - *seq_number = seqr->seq_num; + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "Primary backend in partition module returned a timestamp based seq"); } + + *seq_number = seqr->seq_num; talloc_free(res); return LDB_SUCCESS; } @@ -1019,8 +1018,6 @@ static int partition_sequence_number(struct ldb_module *module, struct ldb_reque int ret; unsigned int i; uint64_t seq_number = 0; - uint64_t timestamp_sequence = 0; - uint64_t timestamp = 0; struct partition_private_data *data = talloc_get_type(ldb_module_get_private(module), struct partition_private_data); struct ldb_seqnum_request *seq; @@ -1098,114 +1095,13 @@ static int partition_sequence_number(struct ldb_module *module, struct ldb_reque } tseqr = talloc_get_type(res->extended->data, struct ldb_seqnum_result); - if (tseqr->flags & LDB_SEQ_TIMESTAMP_SEQUENCE) { - timestamp_sequence = MAX(timestamp_sequence, - tseqr->seq_num); - } else { - seq_number += tseqr->seq_num; - } - talloc_free(res); - } - /* fall through */ - case LDB_SEQ_HIGHEST_TIMESTAMP: - - res = talloc_zero(req, struct ldb_result); - if (res == NULL) { - return ldb_oom(ldb_module_get_ctx(module)); - } - - tseq = talloc_zero(res, struct ldb_seqnum_request); - if (tseq == NULL) { - talloc_free(res); - return ldb_oom(ldb_module_get_ctx(module)); - } - tseq->type = LDB_SEQ_HIGHEST_TIMESTAMP; - - ret = ldb_build_extended_req(&treq, ldb_module_get_ctx(module), res, - LDB_EXTENDED_SEQUENCE_NUMBER, - tseq, - NULL, - res, - ldb_extended_default_callback, - req); - LDB_REQ_SET_LOCATION(treq); - if (ret != LDB_SUCCESS) { - talloc_free(res); - return ret; - } - - ret = ldb_next_request(module, treq); - if (ret != LDB_SUCCESS) { - talloc_free(res); - return ret; - } - ret = ldb_wait(treq->handle, LDB_WAIT_ALL); - if (ret != LDB_SUCCESS) { - talloc_free(res); - return ret; - } - - tseqr = talloc_get_type(res->extended->data, - struct ldb_seqnum_result); - timestamp = tseqr->seq_num; - - talloc_free(res); - - /* Skip the lot if 'data' isn't here yet (initialisation) */ - for (i=0; data && data->partitions && data->partitions[i]; i++) { - - res = talloc_zero(req, struct ldb_result); - if (res == NULL) { - return ldb_oom(ldb_module_get_ctx(module)); - } - - tseq = talloc_zero(res, struct ldb_seqnum_request); - if (tseq == NULL) { - talloc_free(res); - return ldb_oom(ldb_module_get_ctx(module)); - } - tseq->type = LDB_SEQ_HIGHEST_TIMESTAMP; - - ret = ldb_build_extended_req(&treq, ldb_module_get_ctx(module), res, - LDB_EXTENDED_SEQUENCE_NUMBER, - tseq, - NULL, - res, - ldb_extended_default_callback, - req); - LDB_REQ_SET_LOCATION(treq); - if (ret != LDB_SUCCESS) { - talloc_free(res); - return ret; - } - - ret = ldb_request_add_control(treq, - DSDB_CONTROL_CURRENT_PARTITION_OID, - false, data->partitions[i]->ctrl); - if (ret != LDB_SUCCESS) { - talloc_free(res); - return ret; - } - - ret = partition_request(data->partitions[i]->module, treq); - if (ret != LDB_SUCCESS) { - talloc_free(res); - return ret; - } - ret = ldb_wait(treq->handle, LDB_WAIT_ALL); - if (ret != LDB_SUCCESS) { - talloc_free(res); - return ret; - } - - tseqr = talloc_get_type(res->extended->data, - struct ldb_seqnum_result); - timestamp = MAX(timestamp, tseqr->seq_num); - + seq_number += tseqr->seq_num; talloc_free(res); } - break; + + case LDB_SEQ_HIGHEST_TIMESTAMP: + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, "LDB_SEQ_HIGHEST_TIMESTAMP not supported"); } ext = talloc_zero(req, struct ldb_extended); @@ -1220,33 +1116,13 @@ static int partition_sequence_number(struct ldb_module *module, struct ldb_reque ext->oid = LDB_EXTENDED_SEQUENCE_NUMBER; ext->data = seqr; - switch (seq->type) { - case LDB_SEQ_NEXT: - case LDB_SEQ_HIGHEST_SEQ: - - /* Has someone above set a timebase sequence? */ - if (timestamp_sequence) { - seqr->seq_num = (((unsigned long long)timestamp << 24) | (seq_number & 0xFFFFFF)); - } else { - seqr->seq_num = seq_number; - } - - if (timestamp_sequence > seqr->seq_num) { - seqr->seq_num = timestamp_sequence; - seqr->flags |= LDB_SEQ_TIMESTAMP_SEQUENCE; - } - - seqr->flags |= LDB_SEQ_GLOBAL_SEQUENCE; - break; - case LDB_SEQ_HIGHEST_TIMESTAMP: - seqr->seq_num = timestamp; - break; - } - + seqr->seq_num = seq_number; if (seq->type == LDB_SEQ_NEXT) { seqr->seq_num++; } + seqr->flags |= LDB_SEQ_GLOBAL_SEQUENCE; + /* send request done */ return ldb_module_done(req, NULL, ext, LDB_SUCCESS); } diff --git a/source4/dsdb/samdb/ldb_modules/simple_ldap_map.c b/source4/dsdb/samdb/ldb_modules/simple_ldap_map.c index 9ed3b41..9c7599a 100644 --- a/source4/dsdb/samdb/ldb_modules/simple_ldap_map.c +++ b/source4/dsdb/samdb/ldb_modules/simple_ldap_map.c @@ -902,13 +902,10 @@ static int entryuuid_sequence_number(struct ldb_module *module, struct ldb_reque seqr->seq_num++; break; case LDB_SEQ_HIGHEST_TIMESTAMP: - { - seqr->seq_num = (seq_num >> 24); - break; - } + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, "LDB_SEQ_HIGHEST_TIMESTAMP not supported"); } + seqr->flags = 0; - seqr->flags |= LDB_SEQ_TIMESTAMP_SEQUENCE; seqr->flags |= LDB_SEQ_GLOBAL_SEQUENCE; /* send request done */ -- 1.7.2.5 From 8b5d2e36278caa9e6b5a2642891c202c60c5558e Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Mon, 14 Nov 2011 13:52:34 +1100 Subject: [PATCH 3/7] s4-dsdb: Return ldb_result context in dsdb_module_extended The result of the extended operation is now available in the calling routine. Signed-off-by: Andrew Tridgell (cherry picked from commit 422fcbbe7244a0c14aeca5cda7f2a28e2ee821b5) --- source4/dsdb/samdb/ldb_modules/util.c | 23 ++++++++++++++++++++--- 1 files changed, 20 insertions(+), 3 deletions(-) diff --git a/source4/dsdb/samdb/ldb_modules/util.c b/source4/dsdb/samdb/ldb_modules/util.c index cee9ac0..4e0001d 100644 --- a/source4/dsdb/samdb/ldb_modules/util.c +++ b/source4/dsdb/samdb/ldb_modules/util.c @@ -314,14 +314,21 @@ int dsdb_module_guid_by_dn(struct ldb_module *module, struct ldb_dn *dn, struct talloc_free(tmp_ctx); return LDB_SUCCESS; } + + /* a ldb_extended request operating on modules below the current module + + Note that this does not automatically start a transaction. If you + need a transaction the caller needs to start it as needed. */ int dsdb_module_extended(struct ldb_module *module, - const char* oid, void* data, - uint32_t dsdb_flags, - struct ldb_request *parent) + TALLOC_CTX *mem_ctx, + struct ldb_result **_res, + const char* oid, void* data, + uint32_t dsdb_flags, + struct ldb_request *parent) { struct ldb_request *req; int ret; @@ -329,6 +336,10 @@ int dsdb_module_extended(struct ldb_module *module, TALLOC_CTX *tmp_ctx = talloc_new(module); struct ldb_result *res; + if (_res != NULL) { + (*_res) = NULL; + } + res = talloc_zero(tmp_ctx, struct ldb_result); if (!res) { talloc_free(tmp_ctx); @@ -373,9 +384,15 @@ int dsdb_module_extended(struct ldb_module *module, ret = ldb_wait(req->handle, LDB_WAIT_ALL); } + if (_res != NULL && ret == LDB_SUCCESS) { + (*_res) = talloc_steal(mem_ctx, res); + } + talloc_free(tmp_ctx); return ret; } + + /* a ldb_modify request operating on modules below the current module -- 1.7.2.5 From 4506f81bd18a1bcb3a5266ac485a8289689e496c Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Mon, 14 Nov 2011 14:14:58 +1100 Subject: [PATCH 4/7] s4-dsdb: use dsdb_module_extended instead of duplicate code Signed-off-by: Andrew Tridgell (cherry picked from commit 349c54528b82946683290f436eb7220ca59505fe) --- source4/dsdb/samdb/ldb_modules/partition.c | 42 ++++++---------------- source4/dsdb/samdb/ldb_modules/partition_init.c | 2 +- 2 files changed, 13 insertions(+), 31 deletions(-) diff --git a/source4/dsdb/samdb/ldb_modules/partition.c b/source4/dsdb/samdb/ldb_modules/partition.c index 9ab53ae..73dbf1e 100644 --- a/source4/dsdb/samdb/ldb_modules/partition.c +++ b/source4/dsdb/samdb/ldb_modules/partition.c @@ -957,45 +957,27 @@ static int partition_del_trans(struct ldb_module *module) } int partition_primary_sequence_number(struct ldb_module *module, TALLOC_CTX *mem_ctx, - enum ldb_sequence_type type, uint64_t *seq_number) + enum ldb_sequence_type type, uint64_t *seq_number, + struct ldb_request *parent) { int ret; struct ldb_result *res; struct ldb_seqnum_request *tseq; - struct ldb_request *treq; struct ldb_seqnum_result *seqr; - res = talloc_zero(mem_ctx, struct ldb_result); - if (res == NULL) { - return ldb_oom(ldb_module_get_ctx(module)); - } - tseq = talloc_zero(res, struct ldb_seqnum_request); + + tseq = talloc_zero(mem_ctx, struct ldb_seqnum_request); if (tseq == NULL) { - talloc_free(res); return ldb_oom(ldb_module_get_ctx(module)); } tseq->type = type; - ret = ldb_build_extended_req(&treq, ldb_module_get_ctx(module), res, - LDB_EXTENDED_SEQUENCE_NUMBER, - tseq, - NULL, - res, - ldb_extended_default_callback, - NULL); - LDB_REQ_SET_LOCATION(treq); + ret = dsdb_module_extended(module, tseq, &res, + LDB_EXTENDED_SEQUENCE_NUMBER, + tseq, + DSDB_FLAG_NEXT_MODULE, + parent); if (ret != LDB_SUCCESS) { - talloc_free(res); - return ret; - } - - ret = ldb_next_request(module, treq); - if (ret != LDB_SUCCESS) { - talloc_free(res); - return ret; - } - ret = ldb_wait(treq->handle, LDB_WAIT_ALL); - if (ret != LDB_SUCCESS) { - talloc_free(res); + talloc_free(tseq); return ret; } @@ -1008,7 +990,7 @@ int partition_primary_sequence_number(struct ldb_module *module, TALLOC_CTX *mem } *seq_number = seqr->seq_num; - talloc_free(res); + talloc_free(tseq); return LDB_SUCCESS; } @@ -1043,7 +1025,7 @@ static int partition_sequence_number(struct ldb_module *module, struct ldb_reque case LDB_SEQ_NEXT: case LDB_SEQ_HIGHEST_SEQ: - ret = partition_primary_sequence_number(module, req, seq->type, &seq_number); + ret = partition_primary_sequence_number(module, req, seq->type, &seq_number, req); if (ret != LDB_SUCCESS) { return ret; } diff --git a/source4/dsdb/samdb/ldb_modules/partition_init.c b/source4/dsdb/samdb/ldb_modules/partition_init.c index 030f592..b3dc644 100644 --- a/source4/dsdb/samdb/ldb_modules/partition_init.c +++ b/source4/dsdb/samdb/ldb_modules/partition_init.c @@ -397,7 +397,7 @@ int partition_reload_if_required(struct ldb_module *module, return ldb_oom(ldb); } - ret = partition_primary_sequence_number(module, mem_ctx, LDB_SEQ_HIGHEST_SEQ, &seq); + ret = partition_primary_sequence_number(module, mem_ctx, LDB_SEQ_HIGHEST_SEQ, &seq, parent); if (ret != LDB_SUCCESS) { talloc_free(mem_ctx); return ret; -- 1.7.2.5 From 87fc14579546747085964de4175a0f206e15bc86 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Mon, 14 Nov 2011 17:38:04 +1100 Subject: [PATCH 5/7] s4-dsdb: Added metadata to partition module for global sequence number This adds support for global sequence number which is independent of partition information. Signed-off-by: Andrew Tridgell (cherry picked from commit 49926a2ac66b9bbaa50735b78a3bf0385c2cf48d) --- source4/dsdb/samdb/ldb_modules/partition.c | 122 +++++- source4/dsdb/samdb/ldb_modules/partition.h | 7 + source4/dsdb/samdb/ldb_modules/partition_init.c | 6 + .../dsdb/samdb/ldb_modules/partition_metadata.c | 492 ++++++++++++++++++++ source4/dsdb/samdb/ldb_modules/wscript_build | 2 +- 5 files changed, 612 insertions(+), 17 deletions(-) create mode 100644 source4/dsdb/samdb/ldb_modules/partition_metadata.c diff --git a/source4/dsdb/samdb/ldb_modules/partition.c b/source4/dsdb/samdb/ldb_modules/partition.c index 73dbf1e..bc8a760 100644 --- a/source4/dsdb/samdb/ldb_modules/partition.c +++ b/source4/dsdb/samdb/ldb_modules/partition.c @@ -824,6 +824,13 @@ static int partition_start_trans(struct ldb_module *module) ret = partition_reload_if_required(module, data, NULL); if (ret != LDB_SUCCESS) { + ldb_next_del_trans(module); + return ret; + } + + ret = partition_metadata_start_trans(module); + if (ret != LDB_SUCCESS) { + ldb_next_del_trans(module); return ret; } @@ -839,6 +846,7 @@ static int partition_start_trans(struct ldb_module *module) ldb_next_del_trans(data->partitions[i]->module); } ldb_next_del_trans(module); + partition_metadata_del_trans(module); return ret; } } @@ -854,10 +862,9 @@ static int partition_prepare_commit(struct ldb_module *module) unsigned int i; struct partition_private_data *data = talloc_get_type(ldb_module_get_private(module), struct partition_private_data); + int ret; for (i=0; data && data->partitions && data->partitions[i]; i++) { - int ret; - if ((module && ldb_module_flags(ldb_module_get_ctx(module)) & LDB_FLG_ENABLE_TRACING)) { ldb_debug(ldb_module_get_ctx(module), LDB_DEBUG_TRACE, "partition_prepare_commit() -> %s", ldb_dn_get_linearized(data->partitions[i]->ctrl->dn)); @@ -874,7 +881,15 @@ static int partition_prepare_commit(struct ldb_module *module) if ((module && ldb_module_flags(ldb_module_get_ctx(module)) & LDB_FLG_ENABLE_TRACING)) { ldb_debug(ldb_module_get_ctx(module), LDB_DEBUG_TRACE, "partition_prepare_commit() -> (metadata partition)"); } - return ldb_next_prepare_commit(module); + + ret = ldb_next_prepare_commit(module); + if (ret != LDB_SUCCESS) { + return ret; + } + + /* metadata prepare commit must come last, as other partitions could modify + * the database inside the prepare commit method of a module */ + return partition_metadata_prepare_commit(module); } @@ -895,6 +910,11 @@ static int partition_end_trans(struct ldb_module *module) data->in_transaction--; } + ret2 = partition_metadata_end_trans(module); + if (ret2 != LDB_SUCCESS) { + ret = ret2; + } + for (i=0; data && data->partitions && data->partitions[i]; i++) { if ((module && ldb_module_flags(ldb_module_get_ctx(module)) & LDB_FLG_ENABLE_TRACING)) { ldb_debug(ldb_module_get_ctx(module), LDB_DEBUG_TRACE, "partition_end_trans() -> %s", @@ -926,6 +946,11 @@ static int partition_del_trans(struct ldb_module *module) unsigned int i; struct partition_private_data *data = talloc_get_type(ldb_module_get_private(module), struct partition_private_data); + ret = partition_metadata_del_trans(module); + if (ret != LDB_SUCCESS) { + final_ret = ret; + } + for (i=0; data && data->partitions && data->partitions[i]; i++) { if ((module && ldb_module_flags(ldb_module_get_ctx(module)) & LDB_FLG_ENABLE_TRACING)) { ldb_debug(ldb_module_get_ctx(module), LDB_DEBUG_TRACE, "partition_del_trans() -> %s", @@ -957,8 +982,8 @@ static int partition_del_trans(struct ldb_module *module) } int partition_primary_sequence_number(struct ldb_module *module, TALLOC_CTX *mem_ctx, - enum ldb_sequence_type type, uint64_t *seq_number, - struct ldb_request *parent) + enum ldb_sequence_type type, uint64_t *seq_number, + struct ldb_request *parent) { int ret; struct ldb_result *res; @@ -981,8 +1006,8 @@ int partition_primary_sequence_number(struct ldb_module *module, TALLOC_CTX *mem return ret; } - seqr = talloc_get_type(res->extended->data, - struct ldb_seqnum_result); + seqr = talloc_get_type_abort(res->extended->data, + struct ldb_seqnum_result); if (seqr->flags & LDB_SEQ_TIMESTAMP_SEQUENCE) { talloc_free(res); return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, @@ -994,8 +1019,13 @@ int partition_primary_sequence_number(struct ldb_module *module, TALLOC_CTX *mem return LDB_SUCCESS; } -/* FIXME: This function is still semi-async */ -static int partition_sequence_number(struct ldb_module *module, struct ldb_request *req) + +/* + * Older version of sequence number as sum of sequence numbers for each partition + */ +int partition_sequence_number_from_partitions(struct ldb_module *module, + struct ldb_request *req, + struct ldb_extended **ext) { int ret; unsigned int i; @@ -1007,7 +1037,6 @@ static int partition_sequence_number(struct ldb_module *module, struct ldb_reque struct ldb_request *treq; struct ldb_seqnum_request *tseq; struct ldb_seqnum_result *tseqr; - struct ldb_extended *ext; struct ldb_result *res; struct dsdb_partition *p; @@ -1086,17 +1115,17 @@ static int partition_sequence_number(struct ldb_module *module, struct ldb_reque return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, "LDB_SEQ_HIGHEST_TIMESTAMP not supported"); } - ext = talloc_zero(req, struct ldb_extended); - if (!ext) { + *ext = talloc_zero(req, struct ldb_extended); + if (!*ext) { return ldb_oom(ldb_module_get_ctx(module)); } - seqr = talloc_zero(ext, struct ldb_seqnum_result); + seqr = talloc_zero(*ext, struct ldb_seqnum_result); if (seqr == NULL) { - talloc_free(ext); + talloc_free(*ext); return ldb_oom(ldb_module_get_ctx(module)); } - ext->oid = LDB_EXTENDED_SEQUENCE_NUMBER; - ext->data = seqr; + (*ext)->oid = LDB_EXTENDED_SEQUENCE_NUMBER; + (*ext)->data = seqr; seqr->seq_num = seq_number; if (seq->type == LDB_SEQ_NEXT) { @@ -1104,6 +1133,67 @@ static int partition_sequence_number(struct ldb_module *module, struct ldb_reque } seqr->flags |= LDB_SEQ_GLOBAL_SEQUENCE; + return LDB_SUCCESS; +} + + +/* + * Newer version of sequence number using metadata tdb + */ +static int partition_sequence_number(struct ldb_module *module, struct ldb_request *req) +{ + struct partition_private_data *data = talloc_get_type(ldb_module_get_private(module), + struct partition_private_data); + struct ldb_extended *ext; + struct ldb_seqnum_request *seq; + struct ldb_seqnum_result *seqr; + uint64_t seq_number; + struct dsdb_partition *p; + int ret; + + p = find_partition(data, NULL, req); + if (p != NULL) { + /* the caller specified what partition they want the + * sequence number operation on - just pass it on + */ + return ldb_next_request(p->module, req); + } + + seq = talloc_get_type_abort(req->op.extended.data, struct ldb_seqnum_request); + switch (seq->type) { + case LDB_SEQ_NEXT: + ret = partition_metadata_sequence_number_increment(module, &seq_number); + if (ret != LDB_SUCCESS) { + return ret; + } + break; + + case LDB_SEQ_HIGHEST_SEQ: + ret = partition_metadata_sequence_number(module, &seq_number); + if (ret != LDB_SUCCESS) { + return ret; + } + break; + + case LDB_SEQ_HIGHEST_TIMESTAMP: + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "LDB_SEQ_HIGHEST_TIMESTAMP not supported"); + } + + ext = talloc_zero(req, struct ldb_extended); + if (!ext) { + return ldb_module_oom(module); + } + seqr = talloc_zero(ext, struct ldb_seqnum_result); + if (seqr == NULL) { + talloc_free(ext); + return ldb_module_oom(module); + } + ext->oid = LDB_EXTENDED_SEQUENCE_NUMBER; + ext->data = seqr; + + seqr->seq_num = seq_number; + seqr->flags |= LDB_SEQ_GLOBAL_SEQUENCE; /* send request done */ return ldb_module_done(req, NULL, ext, LDB_SUCCESS); diff --git a/source4/dsdb/samdb/ldb_modules/partition.h b/source4/dsdb/samdb/ldb_modules/partition.h index d05ff5d..0cb5705 100644 --- a/source4/dsdb/samdb/ldb_modules/partition.h +++ b/source4/dsdb/samdb/ldb_modules/partition.h @@ -21,6 +21,7 @@ #include #include #include +#include "lib/util/tdb_wrap.h" #include "dsdb/samdb/samdb.h" #include "dsdb/samdb/ldb_modules/util.h" #include "system/locale.h" @@ -39,9 +40,15 @@ struct partition_module { struct ldb_dn *dn; }; +struct partition_metadata { + struct tdb_wrap *db; + int in_transaction; +}; + struct partition_private_data { struct dsdb_partition **partitions; struct ldb_dn **replicate; + struct partition_metadata *metadata; struct partition_module **modules; const char *ldapBackend; diff --git a/source4/dsdb/samdb/ldb_modules/partition_init.c b/source4/dsdb/samdb/ldb_modules/partition_init.c index b3dc644..dc09e13 100644 --- a/source4/dsdb/samdb/ldb_modules/partition_init.c +++ b/source4/dsdb/samdb/ldb_modules/partition_init.c @@ -875,5 +875,11 @@ int partition_init(struct ldb_module *module) return ldb_operr(ldb); } + /* This loads metadata tdb. If it's missing, creates it */ + ret = partition_metadata_init(module); + if (ret != LDB_SUCCESS) { + return ret; + } + return ldb_next_init(module); } diff --git a/source4/dsdb/samdb/ldb_modules/partition_metadata.c b/source4/dsdb/samdb/ldb_modules/partition_metadata.c new file mode 100644 index 0000000..05f87d0 --- /dev/null +++ b/source4/dsdb/samdb/ldb_modules/partition_metadata.c @@ -0,0 +1,492 @@ +/* + Partitions ldb module - management of metadata.tdb for sequence number + + Copyright (C) Amitay Isaacs 2011 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#include "dsdb/samdb/ldb_modules/partition.h" +#include "system/filesys.h" + +#define LDB_METADATA_SEQ_NUM "SEQ_NUM" + + +/* + * Read a key with uint64 value + */ +static int partition_metadata_get_uint64(struct ldb_module *module, + const char *key, uint64_t *value, + uint64_t default_value) +{ + struct partition_private_data *data; + struct tdb_context *tdb; + TDB_DATA tdb_key, tdb_data; + char *value_str; + TALLOC_CTX *tmp_ctx; + + data = talloc_get_type_abort(ldb_module_get_private(module), + struct partition_private_data); + + if (!data && !data->metadata && !data->metadata->db) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: metadata tdb not initialized"); + } + + tmp_ctx = talloc_new(NULL); + if (tmp_ctx == NULL) { + return ldb_module_oom(module); + } + + tdb = data->metadata->db->tdb; + + tdb_key.dptr = (uint8_t *)discard_const_p(char, key); + tdb_key.dsize = strlen(key); + + tdb_data = tdb_fetch(tdb, tdb_key); + if (!tdb_data.dptr) { + if (tdb_error(tdb) == TDB_ERR_NOEXIST) { + *value = default_value; + return LDB_SUCCESS; + } else { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + tdb_errorstr(tdb)); + } + } + + value_str = talloc_strndup(tmp_ctx, (char *)tdb_data.dptr, tdb_data.dsize); + if (value_str == NULL) { + SAFE_FREE(tdb_data.dptr); + talloc_free(tmp_ctx); + return ldb_module_oom(module); + } + + *value = strtoull(value_str, NULL, 10); + + SAFE_FREE(tdb_data.dptr); + talloc_free(tmp_ctx); + + return LDB_SUCCESS; +} + + +/* + * Write a key with uin64 value + */ +static int partition_metadata_set_uint64(struct ldb_module *module, + const char *key, uint64_t value, + bool insert) +{ + struct partition_private_data *data; + struct tdb_context *tdb; + TDB_DATA tdb_key, tdb_data; + int tdb_flag; + char *value_str; + TALLOC_CTX *tmp_ctx; + + data = talloc_get_type_abort(ldb_module_get_private(module), + struct partition_private_data); + + if (!data && !data->metadata && !data->metadata->db) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: metadata tdb not initialized"); + } + + tmp_ctx = talloc_new(NULL); + if (tmp_ctx == NULL) { + return ldb_module_oom(module); + } + + tdb = data->metadata->db->tdb; + + value_str = talloc_asprintf(tmp_ctx, "%llu", (unsigned long long)value); + if (value_str == NULL) { + talloc_free(tmp_ctx); + return ldb_module_oom(module); + } + + tdb_key.dptr = (uint8_t *)discard_const_p(char, key); + tdb_key.dsize = strlen(key); + + tdb_data.dptr = (uint8_t *)value_str; + tdb_data.dsize = strlen(value_str); + + if (insert) { + tdb_flag = TDB_INSERT; + } else { + tdb_flag = TDB_MODIFY; + } + + if (tdb_store(tdb, tdb_key, tdb_data, tdb_flag) != 0) { + talloc_free(tmp_ctx); + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + tdb_errorstr(tdb)); + } + + talloc_free(tmp_ctx); + + return LDB_SUCCESS; +} + + +/* + * Open sam.ldb.d/metadata.tdb. + */ +static int partition_metadata_open(struct ldb_module *module, bool create) +{ + struct ldb_context *ldb = ldb_module_get_ctx(module); + TALLOC_CTX *tmp_ctx; + struct partition_private_data *data; + struct loadparm_context *lp_ctx; + const char *sam_name; + char *filename, *dirname; + int open_flags; + + data = talloc_get_type_abort(ldb_module_get_private(module), + struct partition_private_data); + if (!data || !data->metadata) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: metadata not initialized"); + } + + tmp_ctx = talloc_new(NULL); + if (tmp_ctx == NULL) { + return ldb_module_oom(module); + } + + sam_name = (const char *)ldb_get_opaque(ldb, "ldb_url"); + if (strncmp("tdb://", sam_name, 6) == 0) { + sam_name += 6; + } + if (!sam_name) { + talloc_free(tmp_ctx); + return ldb_operr(ldb); + } + filename = talloc_asprintf(tmp_ctx, "%s.d/metadata.tdb", sam_name); + if (!filename) { + talloc_free(tmp_ctx); + return ldb_oom(ldb); + } + + open_flags = O_RDWR; + if (create) { + open_flags |= O_CREAT; + + /* While provisioning, sam.ldb.d directory may not exist, + * so create it. Ignore errors, if it already exists. */ + dirname = talloc_asprintf(tmp_ctx, "%s.d", sam_name); + if (!dirname) { + talloc_free(tmp_ctx); + return ldb_oom(ldb); + } + + mkdir(dirname, 0700); + talloc_free(dirname); + } + + lp_ctx = talloc_get_type_abort(ldb_get_opaque(ldb, "loadparm"), + struct loadparm_context); + + data->metadata->db = tdb_wrap_open(data->metadata, filename, 10, + TDB_DEFAULT, open_flags, 0660, + lp_ctx); + if (data->metadata->db == NULL) { + talloc_free(tmp_ctx); + if (create) { + ldb_debug(ldb, LDB_DEBUG_ERROR, + "partition_metadata: Unable to create %s", + filename); + } + return LDB_ERR_OPERATIONS_ERROR; + } + + talloc_free(tmp_ctx); + return LDB_SUCCESS; +} + + +/* + * Set the sequence number calculated from older logic (sum of primary sequence + * numbers for each partition) as LDB_METADATA_SEQ_NUM key. + */ +static int partition_metadata_set_sequence_number(struct ldb_module *module) +{ + struct partition_private_data *data; + struct ldb_result *res; + struct ldb_request *req; + struct ldb_seqnum_request *seq; + struct ldb_seqnum_result *seqr; + struct ldb_extended *ext; + TALLOC_CTX *tmp_ctx; + int ret; + uint64_t seq_number; + + data = talloc_get_type_abort(ldb_module_get_private(module), + struct partition_private_data); + if (!data || !data->metadata) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: metadata not initialized"); + } + + tmp_ctx = talloc_new(data->metadata); + if (tmp_ctx == NULL) { + return ldb_module_oom(module); + } + + res = talloc_zero(tmp_ctx, struct ldb_result); + if (res == NULL) { + talloc_free(tmp_ctx); + return ldb_module_oom(module); + } + + seq = talloc_zero(tmp_ctx, struct ldb_seqnum_request); + if (seq == NULL) { + talloc_free(tmp_ctx); + return ldb_module_oom(module); + } + seq->type = LDB_SEQ_HIGHEST_SEQ; + + /* Build an extended request, so it can be passed to each partition in + partition_sequence_number_from_partitions() */ + ret = ldb_build_extended_req(&req, + ldb_module_get_ctx(module), + tmp_ctx, + LDB_EXTENDED_SEQUENCE_NUMBER, + seq, + NULL, + res, + ldb_extended_default_callback, + NULL); + LDB_REQ_SET_LOCATION(req); + if (ret != LDB_SUCCESS) { + talloc_free(tmp_ctx); + return ret; + } + + ret = partition_sequence_number_from_partitions(module, req, &ext); + if (ret != LDB_SUCCESS) { + talloc_free(tmp_ctx); + return ret; + } + + seqr = talloc_get_type_abort(ext->data, struct ldb_seqnum_result); + seq_number = seqr->seq_num; + + talloc_free(tmp_ctx); + + return partition_metadata_set_uint64(module, LDB_METADATA_SEQ_NUM, seq_number, true); +} + + +/* + * Initialize metadata. Load metadata.tdb. + * If missing, create it and fill in sequence number + */ +int partition_metadata_init(struct ldb_module *module) +{ + struct partition_private_data *data; + int ret; + + data = talloc_get_type_abort(ldb_module_get_private(module), + struct partition_private_data); + + data->metadata = talloc_zero(data, struct partition_metadata); + if (data->metadata == NULL) { + return ldb_module_oom(module); + } + + ret = partition_metadata_open(module, false); + if (ret == LDB_SUCCESS) { + goto end; + } + + /* metadata.tdb does not exist, create it */ + DEBUG(2, ("partition_metadata: Migrating partition metadata\n")); + ret = partition_metadata_open(module, true); + if (ret != LDB_SUCCESS) { + talloc_free(data->metadata); + data->metadata = NULL; + goto end; + } + + ret = partition_metadata_set_sequence_number(module); + if (ret != LDB_SUCCESS) { + talloc_free(data->metadata); + data->metadata = NULL; + } + +end: + return ret; +} + + +/* + * Read the sequence number, default to 0 if LDB_METADATA_SEQ_NUM key is missing + */ +int partition_metadata_sequence_number(struct ldb_module *module, uint64_t *value) +{ + return partition_metadata_get_uint64(module, + LDB_METADATA_SEQ_NUM, + value, + 0); +} + + +/* + * Increment the sequence number, returning the new sequence number + */ +int partition_metadata_sequence_number_increment(struct ldb_module *module, uint64_t *value) +{ + struct partition_private_data *data; + int ret; + + data = talloc_get_type_abort(ldb_module_get_private(module), + struct partition_private_data); + if (!data && !data->metadata) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: metadata not initialized"); + } + + if (data->metadata->in_transaction == 0) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: increment sequence number without transaction"); + } + + ret = partition_metadata_get_uint64(module, LDB_METADATA_SEQ_NUM, value, 0); + if (ret != LDB_SUCCESS) { + return ret; + } + + (*value)++; + ret = partition_metadata_set_uint64(module, LDB_METADATA_SEQ_NUM, *value, false); + return ret; +} + + +/* + * Transaction start + */ +int partition_metadata_start_trans(struct ldb_module *module) +{ + struct partition_private_data *data; + struct tdb_context *tdb; + + data = talloc_get_type_abort(ldb_module_get_private(module), + struct partition_private_data); + if (!data && !data->metadata && !data->metadata->db) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: metadata not initialized"); + } + tdb = data->metadata->db->tdb; + + if (tdb_transaction_start(tdb) != 0) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + tdb_errorstr(tdb)); + } + + data->metadata->in_transaction++; + + return LDB_SUCCESS; +} + + +/* + * Transaction prepare commit + */ +int partition_metadata_prepare_commit(struct ldb_module *module) +{ + struct partition_private_data *data; + struct tdb_context *tdb; + + data = talloc_get_type_abort(ldb_module_get_private(module), + struct partition_private_data); + if (!data && !data->metadata && !data->metadata->db) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: metadata not initialized"); + } + tdb = data->metadata->db->tdb; + + if (data->metadata->in_transaction == 0) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: not in transaction"); + } + + if (tdb_transaction_prepare_commit(tdb) != 0) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + tdb_errorstr(tdb)); + } + + return LDB_SUCCESS; +} + + +/* + * Transaction end + */ +int partition_metadata_end_trans(struct ldb_module *module) +{ + struct partition_private_data *data; + struct tdb_context *tdb; + + data = talloc_get_type_abort(ldb_module_get_private(module), + struct partition_private_data); + if (!data && !data->metadata && !data->metadata->db) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: metadata not initialized"); + } + tdb = data->metadata->db->tdb; + + if (data->metadata->in_transaction == 0) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: not in transaction"); + } + + data->metadata->in_transaction--; + + if (tdb_transaction_commit(tdb) != 0) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + tdb_errorstr(tdb)); + } + + return LDB_SUCCESS; +} + + +/* + * Transaction delete + */ +int partition_metadata_del_trans(struct ldb_module *module) +{ + struct partition_private_data *data; + struct tdb_context *tdb; + + data = talloc_get_type_abort(ldb_module_get_private(module), + struct partition_private_data); + if (!data && !data->metadata && !data->metadata->db) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: metadata not initialized"); + } + tdb = data->metadata->db->tdb; + + if (data->metadata->in_transaction == 0) { + return ldb_module_error(module, LDB_ERR_OPERATIONS_ERROR, + "partition_metadata: not in transaction"); + } + + data->metadata->in_transaction--; + + tdb_transaction_cancel(tdb); + + return LDB_SUCCESS; +} diff --git a/source4/dsdb/samdb/ldb_modules/wscript_build b/source4/dsdb/samdb/ldb_modules/wscript_build index 87e915d..7f74ea4 100644 --- a/source4/dsdb/samdb/ldb_modules/wscript_build +++ b/source4/dsdb/samdb/ldb_modules/wscript_build @@ -194,7 +194,7 @@ bld.SAMBA_MODULE('ldb_show_deleted', bld.SAMBA_MODULE('ldb_partition', - source='partition.c partition_init.c', + source='partition.c partition_init.c partition_metadata.c', autoproto='partition_proto.h', subsystem='ldb', init_function='ldb_partition_module_init', -- 1.7.2.5 From 2ac6897236b56cae291b398e07872ae699af2f2d Mon Sep 17 00:00:00 2001 From: root Date: Mon, 5 Dec 2011 13:26:41 +0100 Subject: [PATCH 6/7] fix the build after: 01c934c81e55b79601122d8e0740c7946077c37e Author: Andrew Bartlett Date: Wed Oct 12 23:01:08 2011 +1100 lib/util: Add back control of mmap and hash size in tdb for top level build This passes down a struct loadparm_context to allow these parameters to be checked. This may be s3 or s4 context, allowing the #if _SAMBA_BUILD_ macro to go away safely. Andrew Bartlett --- .../dsdb/samdb/ldb_modules/partition_metadata.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diff --git a/source4/dsdb/samdb/ldb_modules/partition_metadata.c b/source4/dsdb/samdb/ldb_modules/partition_metadata.c index 05f87d0..62e09d8 100644 --- a/source4/dsdb/samdb/ldb_modules/partition_metadata.c +++ b/source4/dsdb/samdb/ldb_modules/partition_metadata.c @@ -199,8 +199,7 @@ static int partition_metadata_open(struct ldb_module *module, bool create) struct loadparm_context); data->metadata->db = tdb_wrap_open(data->metadata, filename, 10, - TDB_DEFAULT, open_flags, 0660, - lp_ctx); + TDB_DEFAULT, open_flags, 0660); if (data->metadata->db == NULL) { talloc_free(tmp_ctx); if (create) { -- 1.7.2.5 From 677996cf1f1b44ad468c01d171c5a45a7182f271 Mon Sep 17 00:00:00 2001 From: Amitay Isaacs Date: Mon, 28 Nov 2011 17:19:50 +1100 Subject: [PATCH 7/7] s4-samdb: seqence_number() operation must be in a transaction (cherry picked from commit 13545d781b257afe3c3a107a669851c38cdfefbd) --- source4/scripting/python/samba/samdb.py | 14 ++++++++++++++ 1 files changed, 14 insertions(+), 0 deletions(-) diff --git a/source4/scripting/python/samba/samdb.py b/source4/scripting/python/samba/samdb.py index 5cceb06..41c4f40 100644 --- a/source4/scripting/python/samba/samdb.py +++ b/source4/scripting/python/samba/samdb.py @@ -817,3 +817,17 @@ accountExpires: %u if sd: m["nTSecurityDescriptor"] = ndr_pack(sd) self.add(m) + + def sequence_number(self, seq_type): + """Returns the value of the sequence number according to the requested type + :param seq_type: type of sequence number + """ + self.transaction_start() + try: + seq = super(SamDB, self).sequence_number(seq_type) + except Exception: + self.transaction_cancel() + raise + else: + self.transaction_commit() + return seq -- 1.7.2.5