2023-02-27

lightning mdb

##emphasis: draft

symas lightning memory-mapped database (lmdb) most examples are currently in sc

external

official api documentation

explanations

environment

a collection of btrees, or alternatively, everything that one database file represents prefix of api routines: mdb_env_

transactions

transactions track the changes you want to make when a transaction is commited, all changes done with the transaction are applied if a transaction is aborted, no changes are applied prefix of api routines: mdb_txn_

reading

when the only thing you do with a transaction is reading, the transaction can be finalised with mdb_txn_abort

mdb-val

stores size and pointers to data of what is to be transferred in or out of the database it is used for keys and values an MDB-val object has two fields: mv_size and mv_data

(define example-value MDB-val)
(define example-key MDB-val)
(define key b32 1234)
(define-array data b8 3 5 6 7)
(struct-set example-key mv_size (sizeof int))
(struct-set example-key mv_data (address-of key))
(struct-set example-value mv_size 4)
(struct-set example-value mv_data (address-of data))

dupsort

keys are sorted and have one value by default, but can have multiple values that are also sorted whenthe MDB-DUPSORT flag is specified when opening a dbi

how to

create and open an environment

(mdb-require-success (mdb-env-create (address-of env)))
; you are free to set database properties afterwards
(mdb-env-set-maxdbs env 7)
(mdb-env-set-mapsize env 1048576000) (mdb-env-set-maxreaders env 100000)
; open
(define flags-env-open b32 (bit-or MDB-NOSUBDIR MDB-NOTLS MDB-WRITEMAP MDB-MAPASYNC))
(mdb-require-success (mdb-env-open env (string-concat path "metadata") flags-env-open 384))
(if (not (= MDB-SUCCESS status)) (begin (mdb-env-close env) (goto bad-status)))

create and open a tree

(define db-options b32 MDB-CREATE) txn-write-begin
(mdb-dbi-open txn "ide->data" db-options (address-of dbi-ide->data))
txn-write-commit

set a custom comparison function

; the custom routine
(define (cmp-ide a b) ((static int) MDB-val* MDB-val*)
  (return
    (if* (< (deref (convert-type (struct-ref (deref a) mv-data) b32*))
      (deref (convert-type (struct-ref (deref b) mv-data) b32*)))
      -1
      (> (deref (convert-type (struct-ref (deref a) mv-data) b32*))
        (deref (convert-type (struct-ref (deref b) mv-data) b32*))))))
; set it for keys
(mdb-set-compare txn dbi-left->pair-data (convert-type cmp-ide MDB-cmp-func*))
; or set it for values
(mdb-set-dupsort txn dbi-left->pair-data (convert-type cmp-pair-data MDB-cmp-func*))

create a cursor

a cursor can be positioned at an element in the database and moved to other elements

(define mycursor MDB-cursor*)
(define status b32_s (mdb-cursor-open txn dbi-mydbi (address-of mycursor)))
(if (not (= MDB-SUCCESS status)) (goto bad-status))

read using a cursor

(mdb-cursor-get pair->ide (address-of val-pair) (address-of val-ide) MDB-SET)

write using a cursor read/write without a cursor

iterate over all keys

(define-macro (mdb-each-entry dbi address-val-key address-val-value body)
  (set status (mdb-cursor-get dbi address-val-key address-val-value MDB-FIRST))
  (while (= MDB-SUCCESS status) body
    (set status (mdb-cursor-get dbi address-val-key address-val-value MDB-NEXT)))
  (mdb-require-notfound status))

close the database

; close trees
(mdb-dbi-close env dbi-ide->data)
; close the environment
(mdb-env-close env)

get error description strings

(mdb-strerror error-number)

abort the transaction of a cursor

when only the cursor is available

(mdb-txn-abort (mdb-cursor-txn cursor)))

minimal example

compile with "gcc -llmdb mdb-example.c"

#include <stdio.h>
#include <lmdb.h>
#define s(arg) status=arg;if(!(MDB_SUCCESS==status)){puts(mdb_strerror(status));return(status);}
int status;
MDB_env* env;
int main() {
  MDB_stat stat;
  int data;
  MDB_val val;
  MDB_txn* txn;
  MDB_dbi dbi;
  s(mdb_env_create(&env));
  s(mdb_env_set_maxdbs(env,10u));
  s(mdb_env_set_maxreaders(env,3u));
  s(mdb_env_open(env,"/tmp",0u,384u));
  s(mdb_txn_begin(env,0u,0u,&txn));
  s(mdb_dbi_open(txn,"testdb",MDB_CREATE,&dbi));
  val.mv_size=sizeof(int);
  val.mv_data=&data;
  data=123u;
  s(mdb_put(txn,dbi,&val,&val,0u));
  data=124u;
  s(mdb_put(txn,dbi,&val,&val,0u));
  data=125u;
  s(mdb_put(txn,dbi,&val,&val,0u));
  s(mdb_txn_commit(txn));
  s(mdb_txn_begin(env,0u,MDB_RDONLY,&txn));
  s(mdb_stat(txn,dbi,&stat));
  mdb_txn_abort(txn);
  printf("ms-entries: %lu\nms-depth: %lu\n",stat.ms_entries,stat.ms_depth);
  mdb_env_close(env);
  return(0);
}

other

some macros i have found helpful

(define-macro (mdb-require-success expr) (set status expr) (mdb-require-success-check status))
(define-macro (mdb-require-notfound expr) (set status expr) (mdb-require-notfound-check status))

(define-macro txn-begin
  (begin (define txn MDB-txn*)
    (mdb-require-success (mdb-txn-begin env 0 MDB-RDONLY (address-of txn)))))

(define-macro txn-write-begin (define txn MDB-txn*)
  (mdb-require-success (mdb-txn-begin env 0 0 (address-of txn))))

(define-macro txn-abort (mdb-txn-abort txn))
(define-macro txn-write-commit (mdb-txn-commit txn))