# copyright (C) 1997-2006 Jean-Luc Fontaine (mailto:jfontain@free.fr)
# this program is free software: please read the COPYRIGHT file enclosed in this package or use the Help Copyright menu

# $Id: database.tcl,v 1.99 2006/04/06 23:07:30 jfontain Exp $

# database interface: SQLite is single file based, MySQL is natively supported, the other engines by ODBC.


class database {

    set (dateTimeFormat) {%Y-%m-%d %T}
    set (includeSamples) 0

    # note: error messages are not internationalized since they can be written to the system log, and debug messages are for english
    # speaking technical people
    proc database {this args} switched {$args} {
        set ($this,error) {}
        switched::complete $this
        set ($this,rowType) BIGINT                                                                                    ;# 64 bits row
        if {$switched::($this,-file) ne ""} {                                              ;# SQLite file based single user database
            set ($this,file) 1
            set ($this,odbc) 0
            if {[catch {package require sqlite3} result3] && [catch {package require sqlite 2} result2]} {    ;# try version 3 first
                set ($this,error) "SQLite interface error:\n$result3\n$result2"
                return
            } elseif {$switched::($this,-debuglevel)} {
                set message "loaded SQLite library version "
                if {[info exists result3]} {append message $result3} else {append message $result2}
if {$global::withGUI} { ;# >8
                puts $message
} else { ;# >8
                writeLog $message debug
} ;# >8
            }
            if {[catch {package present sqlite3}]} {set ($this,sqliteExtension) {}} else {set ($this,sqliteExtension) 3}
            sqliteOpen $this                                                                         ;# can fail on misconfiguration
            if {$($this,error) ne ""} return                                          ;# abort so that object can be cleanly deleted
        } elseif {$switched::($this,-dsn) ne ""} {                 ;# ODBC (moodss (or other name) database must be already created)
            set ($this,file) 0
            set ($this,odbc) 1
            if {[catch {package require tclodbc 2} result]} {
                set ($this,error) "Tcl ODBC interface error: $result"
                return
            }
            foreach list [::database datasources] {
                foreach {dsn driver} $list {}
                if {$dsn eq $switched::($this,-dsn)} break                                                           ;# found driver
            }
            if {$switched::($this,-debuglevel)} {
                set message "loaded ODBC driver $driver version $result"
if {$global::withGUI} { ;# >8
                puts $message
} else { ;# >8
                writeLog $message debug
} ;# >8
            }
            odbcConnect $this                                                                        ;# can fail on misconfiguration
            if {$($this,error) ne ""} return                                          ;# abort so that object can be cleanly deleted
            set ($this,limit) {LIMIT %l}
            set ($this,prefix) {}                                                                      ;# columns name common prefix
            set ($this,timeStamp) TIMESTAMP
            set ($this,text) TEXT                                                                                ;# column text type
            switch -glob [string tolower $driver] {
                *my* {                                                                  ;# matches MySQL, /usr/lib/libmyodbc.so, ...
                    set ($this,timeStamp) DATETIME           ;# safer as not changed by manual updates, changes, ... of the database
                    set ($this,lock) {LOCK TABLES %t WRITE}
                    set ($this,unlock) {UNLOCK TABLES}
                    set ($this,type) mysql
                }
                *postg* - *psql* - *pgsql* {                                                                           ;# PostgreSQL
                    set ($this,type) postgres
                }
                *db2* {                                                                                                       ;# DB2
                    set ($this,limit) {FETCH FIRST %l ROWS ONLY}
                    set ($this,type) db2
                }
                *ora* {                                                                                                    ;# Oracle
                    set ($this,prefix) c                                      ;# to work around the many limitations on column names
                    set ($this,timeStamp) DATE
                    set ($this,text) VARCHAR2(4000)
                    set ($this,rowType) INTEGER
                    set ($this,type) oracle
                }
                *sql*server* {
                    set ($this,timeStamp) DATETIME
                    set ($this,type) mssql
                }
            }
        } elseif {$switched::($this,-database) ne ""} {                                                                     ;# MySQL
            set ($this,file) 0
            set ($this,odbc) 0
            if {[catch {package require mysqltcl} result]} {
                set ($this,error) "Tcl MySQL interface error: $result"
                return
            } elseif {$switched::($this,-debuglevel)} {
                set message "loaded MySQL native driver version $result"
if {$global::withGUI} { ;# >8
                puts $message
} else { ;# >8
                writeLog $message debug
} ;# >8
            }
            mysqlConnect $this                                                                       ;# can fail on misconfiguration
            if {$($this,error) ne ""} return                                          ;# abort so that object can be cleanly deleted
            set ($this,lock) {LOCK TABLES %t WRITE}
            set ($this,unlock) {UNLOCK TABLES}
        } else {
            set ($this,file) 0
            set ($this,odbc) 0
            set ($this,error) [mc "No database defined: in preferences, configure application database"]
            return
        }
        initialize $this
        if {$($this,error) ne ""} return                                                   ;# abort on fatal error in initialization
        checkFormat $this
        if {$($this,error) ne ""} return                                                  ;# abort on fatal error in format checking
if {$global::withGUI} { ;# >8
        # upper time limit for all queries since database may be constantly written to (so the cache can function properly)
        set ($this,start) [dateTime $this]
} else { ;# >8
        if {$($this,oldFormat)} {
            # only reading from an old format database is implemented (writing requires new format starting at moomps 2.3)
            set ($this,error) {cannot write to a database in old format (see upgrading section in database documentation)}
        }
} ;# >8
    }

    proc ~database {this} {
        variable ${this}cache

        unset -nocomplain ${this}cache
        disconnect $this
    }

    proc options {this} {                                                                 ;# force initialization of index and label
        # -database and -port options are solely for MySQL, -file solely for SQLite, -dsn solely for ODBC
        return [list\
            [list -database {} {}]\
            [list -debuglevel 0 0]\
            [list -dsn {} {}]\
            [list -file {} {}]\
            [list -host {} {}]\
            [list -password {} {}]\
            [list -port {} {}]\
            [list -readonly 0 0]\
            [list -user $::tcl_platform(user) $::tcl_platform(user)]\
        ]
    }

    foreach option {-database -dsn -file -host -password -port -readonly -user} {
        proc set$option {this value} "
            if {\$switched::(\$this,complete)} {
                ::error {option $option cannot be set dynamically}
            }
        "
    }

    proc set-debuglevel {this value} {            ;# at this time, there are no debug levels, so level should be viewed as a boolean
        if {$switched::($this,complete)} {
            ::error {option -debuglevel cannot be set dynamically}
        }
    }

if {$global::withGUI} { ;# >8

    # note: use error data member so that error occurence can be detected by its non emptiness

    proc errorTrace {this message} {
        if {[info exists ($this,ignoreErrors)]} return
        if {$switched::($this,-debuglevel)} {puts $message}
        residentTraceModule 1
        modules::trace {} moodss(database) $message
    }
    proc sqliteOpen {this} {
        set ($this,error) {}
        unset -nocomplain ($this,connection)
        if {$switched::($this,-debuglevel)} {puts "opening file \"$switched::($this,-file)\""}
        set connection sqlite$this                                                                   ;# use a unique connection name
        if {[catch {::sqlite$($this,sqliteExtension) $connection $switched::($this,-file)} message]} {
            errorTrace $this [set ($this,error) $message]
        } else {
            if {$switched::($this,-readonly)} {
                if {$switched::($this,-debuglevel)} {puts {setting lock timeout at 1 minute}}
                $connection timeout 60000                    ;# so that GUI does not fail while recording is done by another process
            }
            set ($this,connection) $connection                                                              ;# successful connection
        }
    }
    proc sqliteEvaluate {this sql} {
        if {![info exists ($this,connection)]} return                                ;# could happen if an error occurred previously
        set ($this,error) {}
        if {$switched::($this,-debuglevel)} {puts [string trim $sql]}
        if {[catch {$($this,connection) eval $sql} result]} {
            errorTrace $this [set ($this,error) $result]
            return {}
        } else {
            return $result
        }
    }
    proc sqliteClose {this} {
        set ($this,error) {}
        if {![info exists ($this,connection)]} return                                                               ;# not connected
        if {$switched::($this,-debuglevel)} {puts "closing file \"$switched::($this,-file)\""}
        if {[catch {$($this,connection) close} message]} {
            errorTrace $this [set ($this,error) $message]
        } else {
            unset ($this,connection)
        }
    }
    proc odbcConnect {this} {
        set ($this,error) {}
        unset -nocomplain ($this,connection)
        if {$switched::($this,-debuglevel)} {puts "connecting to ODBC DSN $switched::($this,-dsn)"}
        set arguments [list $switched::($this,-dsn)]
        if {$switched::($this,-user) ne ""} {lappend arguments $switched::($this,-user)}
        if {$switched::($this,-password) ne ""} {lappend arguments $switched::($this,-password)}
        if {[catch {set connection [eval ::database odbc$this $arguments]} message]} {               ;# use a unique connection name
            errorTrace $this [set ($this,error) $message]
        } else {
            set ($this,connection) $connection                                                              ;# successful connection
        }
    }
    proc odbcConnection {this args} {
        if {![info exists ($this,connection)]} return                                ;# could happen if an error occurred previously
        set ($this,error) {}
        if {$switched::($this,-debuglevel)} {puts "[eval concat $args]"}
        if {[catch {eval $($this,connection) $args} result]} {
            errorTrace $this [set ($this,error) $result]
            return {}
        } else {
            return $result
        }
    }
    proc odbcDisconnect {this} {
        set ($this,error) {}
        if {![info exists ($this,connection)]} return                                                               ;# not connected
        if {$switched::($this,-debuglevel)} {puts {closing ODBC connection}}
        if {[catch {$($this,connection) disconnect} message]} {
            errorTrace $this [set ($this,error) $message]
        } else {
            unset ($this,connection)
        }
    }
    proc mysqlConnect {this} {
        set ($this,error) {}
        unset -nocomplain ($this,connection)
        if {$switched::($this,-debuglevel)} {puts "connecting to database $switched::($this,-database)"}
        set arguments [list -db $switched::($this,-database)]                                                 ;# database must exist
        if {$switched::($this,-host) ne ""} {lappend arguments -host $switched::($this,-host)}
        if {$switched::($this,-user) ne ""} {lappend arguments -user $switched::($this,-user)}
        if {$switched::($this,-password) ne ""} {lappend arguments -password $switched::($this,-password)}
        if {$switched::($this,-port) ne ""} {lappend arguments -port $switched::($this,-port)}
        if {[catch {set connection [eval mysqlconnect $arguments]} message]} {
            errorTrace $this [set ($this,error) $message]
        } else {
            set ($this,connection) $connection                                                              ;# successful connection
        }
    }
    proc mysqlSelect {this args} {
        set ($this,error) {}
        if {$switched::($this,-debuglevel)} {puts [string trim [lindex $args 0]]}                ;# only keep query, without options
        if {[catch {eval mysqlsel $($this,connection) $args} result]} {
            errorTrace $this [set ($this,error) $result]
            return {}
        } else {
            return $result
        }
    }
    proc mysqlExecute {this args} {
        if {![info exists ($this,connection)]} return                                ;# could happen if an error occurred previously
        set ($this,error) {}
        if {$switched::($this,-debuglevel)} {puts [string trim [lindex $args 0]]}            ;# only keep statement, without options
        if {[catch {eval mysqlexec $($this,connection) $args} message]} {
            errorTrace $this [set ($this,error) $message]
        }
    }
    proc mysqlColumns {this args} {
        set ($this,error) {}
        if {$switched::($this,-debuglevel)} {puts "SHOW COLUMNS FROM [lindex $args 0]"}                                     ;# table
        if {[catch {eval mysqlcol $($this,connection) $args} result]} {
            errorTrace $this [set ($this,error) $result]
            return {}
        } else {
            return $result
        }
    }
    proc mysqlDisconnect {this} {
        set ($this,error) {}
        if {![info exists ($this,connection)]} return                                                               ;# not connected
        if {$switched::($this,-debuglevel)} {puts {closing MySQL connection}}
        if {[catch {mysqlclose $($this,connection)} message]} {
            errorTrace $this [set ($this,error) $message]
        } else {
            unset ($this,connection)
        }
    }

} else { ;# >8

    proc error {this} {                         ;# use a procedure since daemon interpreters have no access to database data members
        return $($this,error)
    }
    proc errorLog {this message} {
        if {[info exists ($this,ignoreErrors)]} return
        writeLog $message error
    }
    proc sqliteOpen {this} {
        set ($this,error) {}
        unset -nocomplain ($this,connection)
        if {$switched::($this,-debuglevel)} {writeLog "opening file \"$switched::($this,-file)\"" debug}
        set connection sqlite$this                                                                   ;# use a unique connection name
        if {[catch {::sqlite$($this,sqliteExtension) $connection $switched::($this,-file)} message]} {
            errorLog $this [set ($this,error) $message]
        } else {
            set ($this,connection) $connection                                                              ;# successful connection
        }
    }
    proc sqliteEvaluate {this sql} {
        if {![info exists ($this,connection)]} return                                ;# could happen if an error occurred previously
        set ($this,error) {}
        if {$switched::($this,-debuglevel)} {writeLog [string trim $sql] debug}
        if {[catch {$($this,connection) eval $sql} result]} {
            errorLog $this [set ($this,error) $result]
            return {}
        } else {
            return $result
        }
    }
    proc sqliteClose {this} {
        set ($this,error) {}
        if {![info exists ($this,connection)]} return                                                               ;# not connected
        if {$switched::($this,-debuglevel)} {writeLog "closing file \"$switched::($this,-file)\"" debug}
        if {[catch {$($this,connection) close} message]} {
            errorLog $this [set ($this,error) $message]
        } else {
            unset ($this,connection)
        }
    }
    proc odbcConnect {this} {
        set ($this,error) {}
        unset -nocomplain ($this,connection)
        if {$switched::($this,-debuglevel)} {writeLog "connecting to ODBC DSN $switched::($this,-dsn)" debug}
        set arguments [list $switched::($this,-dsn)]
        if {$switched::($this,-user) ne ""} {lappend arguments $switched::($this,-user)}
        if {$switched::($this,-password) ne ""} {lappend arguments $switched::($this,-password)}
        if {[catch {set connection [eval ::database odbc$this $arguments]} message]} {               ;# use a unique connection name
            errorLog $this [set ($this,error) $message]
        } else {
            set ($this,connection) $connection                                                              ;# successful connection
        }
    }
    proc odbcConnection {this args} {
        if {![info exists ($this,connection)]} return                                ;# could happen if an error occurred previously
        set ($this,error) {}
        if {$switched::($this,-debuglevel)} {writeLog "[eval concat $args]" debug}
        if {[catch {eval $($this,connection) $args} result]} {
            errorLog $this [set ($this,error) $result]
            return {}
        } else {
            return $result
        }
    }
    proc odbcDisconnect {this} {
        set ($this,error) {}
        if {![info exists ($this,connection)]} return                                                               ;# not connected
        if {$switched::($this,-debuglevel)} {writeLog {closing ODBC connection} debug}
        if {[catch {$($this,connection) disconnect} message]} {
            errorLog $this [set ($this,error) $message]
        } else {
            unset ($this,connection)
        }
    }
    proc mysqlConnect {this} {
        set ($this,error) {}
        unset -nocomplain ($this,connection)
        if {$switched::($this,-debuglevel)} {writeLog "connecting to database $switched::($this,-database)" debug}
        set arguments [list -db $switched::($this,-database)]                                                 ;# database must exist
        if {$switched::($this,-host) ne ""} {lappend arguments -host $switched::($this,-host)}
        if {$switched::($this,-user) ne ""} {lappend arguments -user $switched::($this,-user)}
        if {$switched::($this,-password) ne ""} {lappend arguments -password $switched::($this,-password)}
        if {$switched::($this,-port) ne ""} {lappend arguments -port $switched::($this,-port)}
        if {[catch {set connection [eval mysqlconnect $arguments]} message]} {
            errorLog $this [set ($this,error) $message]
        } else {
            set ($this,connection) $connection                                                              ;# successful connection
        }
    }
    proc mysqlSelect {this args} {
        set ($this,error) {}
        if {$switched::($this,-debuglevel)} {writeLog [string trim [lindex $args 0]] debug}
        if {[catch {eval mysqlsel $($this,connection) $args} result]} {
            errorLog $this [set ($this,error) $result]
            return {}
        } else {
            return $result
        }
    }
    proc mysqlExecute {this args} {
        if {![info exists ($this,connection)]} return                                ;# could happen if an error occurred previously
        set ($this,error) {}
        if {$switched::($this,-debuglevel)} {writeLog [string trim [lindex $args 0]] debug}
        if {[catch {eval mysqlexec $($this,connection) $args} message]} {
            errorLog $this [set ($this,error) $message]
        }
    }
    proc mysqlColumns {this args} {
        set ($this,error) {}
        if {$switched::($this,-debuglevel)} {writeLog "SHOW COLUMNS FROM [lindex $args 0]" debug}
        if {[catch {eval mysqlcol $($this,connection) $args} result]} {
            errorLog $this [set ($this,error) $result]
            return {}
        } else {
            return $result
        }
    }
    proc mysqlDisconnect {this} {
        set ($this,error) {}
        if {![info exists ($this,connection)]} return                                                               ;# not connected
        if {$switched::($this,-debuglevel)} {writeLog {closing MySQL connection} debug}
        if {[catch {mysqlclose $($this,connection)} message]} {
            errorLog $this [set ($this,error) $message]
        } else {
            unset ($this,connection)
        }
    }

    proc 64Bits {this} {
        return $($this,64Bits)
    }

} ;# >8

    proc initialize {this} {                 ;# create tables if they do not exist (database must always have been manually created)
        set file $($this,file)
        set odbc $($this,odbc)
        if {$file} {
            set timeStamp INTEGER                                         ;# store seconds since SQLite has no native date time type
            set prefix {}
            set text TEXT
        } elseif {$odbc} {
            set timeStamp $($this,timeStamp)
            set prefix $($this,prefix)
            set text $($this,text)
        } else {                                                                                                            ;# MySQL
            set timeStamp DATETIME                           ;# safer as not changed by manual updates, changes, ... of the database
            set prefix {}
            set text TEXT
        }
        set rowType $($this,rowType)                                                                              ;# row column type
        # see tables documentation in moomps documentation
        array set statements "
            instances {
                {
                    CREATE TABLE instances (
                        ${prefix}number INTEGER NOT NULL PRIMARY KEY,
                        ${prefix}start $timeStamp NOT NULL,
                        ${prefix}module VARCHAR(255) NOT NULL,
                        ${prefix}identifier VARCHAR(255),
                        ${prefix}major INTEGER NOT NULL,
                        ${prefix}minor INTEGER NOT NULL
                    )
                }
            } options {
                {
                    CREATE TABLE options (
                        ${prefix}instance INTEGER NOT NULL REFERENCES instances,
                        ${prefix}name VARCHAR(255) NOT NULL,
                        ${prefix}value $text
                    )
                }
            } entries {
                {
                    CREATE TABLE entries (
                        ${prefix}instance INTEGER NOT NULL REFERENCES instances,
                        ${prefix}number INTEGER NOT NULL,
                        ${prefix}indexed INTEGER NOT NULL,
                        ${prefix}label VARCHAR(255) NOT NULL,
                        ${prefix}type VARCHAR(16) NOT NULL,
                        ${prefix}message $text NOT NULL,
                        ${prefix}anchor VARCHAR(16),
                        UNIQUE(${prefix}instance, ${prefix}number)
                    )
                }
            } history {
                {
                    CREATE TABLE history (
                        ${prefix}instant $timeStamp NOT NULL,
                        ${prefix}instance INTEGER NOT NULL REFERENCES instances,
                        ${prefix}row $rowType NOT NULL,
                        ${prefix}entry INTEGER NOT NULL,
                        ${prefix}value VARCHAR(255)
                    )
                } {
                    CREATE INDEX ${prefix}cell ON history (${prefix}instance, ${prefix}row, ${prefix}entry)
                }
            } data {
                {
                    CREATE TABLE data (
                        ${prefix}instance INTEGER NOT NULL REFERENCES instances,
                        ${prefix}row $rowType NOT NULL,
                        ${prefix}entry INTEGER NOT NULL,
                        ${prefix}label VARCHAR(255) NOT NULL,
                        ${prefix}comment VARCHAR(255),
                        UNIQUE(${prefix}instance, ${prefix}row, ${prefix}entry)
                    )
                }
            }
        "
        if {!$file} {                   ;# SQLite is too picky about index names are not likely to hold a huge amount of data anyway
            lappend statements(options) "CREATE INDEX ${prefix}instance ON options (${prefix}instance)"
        }
        set ($this,created) 0                            ;# public flag so that we know whether one or more tables were just created
        foreach table {instances options entries history data} {                           ;# in proper order for references to work
            set query "SELECT COUNT(*) FROM $table"
            set ($this,ignoreErrors) {}                                         ;# just for the time when table existence is checked
            if {$file} {
                sqliteEvaluate $this $query
            } elseif {$odbc} {
                odbcConnection $this $query
            } else {
                mysqlSelect $this $query
            }
            unset ($this,ignoreErrors)
            if {$($this,error) eq ""} continue                                                               ;# table exists already
if {$global::withGUI} { ;# >8
            lifoLabel::push $global::messenger [format [mc {creating database table %s...}] $table]
            busy 1 .
} ;# >8
            foreach statement $statements($table) {
                if {$file} {
                    sqliteEvaluate $this $statement
                } elseif {$odbc} {
                    odbcConnection $this $statement
                } else {
                    mysqlExecute $this $statement
                }
                if {$($this,error) ne ""} break                                ;# fatal error, abort, as error is handled by invoker
            }
if {$global::withGUI} { ;# >8
            busy 0 .
            lifoLabel::pop $global::messenger
} ;# >8
            set ($this,created) [expr {$($this,error) eq ""}]
            if {$($this,error) ne ""} break                                    ;# fatal error, abort, as error is handled by invoker
        }
    }

if {$global::withGUI} { ;# >8

    proc initializeSamples {install} {                                                                                 ;# idempotent
        variable sampleModule

        unset -nocomplain sampleModule
        if {$install} {
            package require datasets
            set (includeSamples) 1
            datasets::initialize $(dateTimeFormat) $global::32BitIntegerMinimum $global::32BitIntegerMaximum
            # use huge numbers not to interfere with real database module instances (but not 64 bits as lsort does not support it)
            set instance $global::32BitIntegerMaximum
            foreach name $datasets::list {                                           ;# simulate module instance storage in database
                incr instance -1
                set sampleModule($instance) $name
            }
        } else {
            set (includeSamples) 0
        }
    }

} ;# >8

    proc checkFormat {this} {                                ;# check columns for the tables that have changed at moomps version 2.3
        if {$($this,file)} {
            set ($this,oldFormat) 0
            set ($this,64Bits) 1                                                                      ;# SQLite columns are typeless
            return                                                                                      ;# SQLite support came later
        }
        if {$($this,odbc)} {                     ;# "$connection columns $table" is not supported by PostgreSQL, so use brute method
            set prefix $($this,prefix)
            set ($this,ignoreErrors) {}
            odbcConnection $this "SELECT COUNT(${prefix}identifier) FROM instances"
            if {$($this,error) eq ""} {set instances(identifier) {}}
            odbcConnection $this "SELECT COUNT(${prefix}instance) FROM entries"
            if {$($this,error) eq ""} {set entries(instance) {}}
            unset ($this,ignoreErrors)
        } else {                                                                                              ;# MySQL native driver
            foreach table {instances entries} {
                set columns [mysqlColumns $this $table name]
                if {$($this,error) ne ""} return                               ;# fatal error, abort, as error is handled by invoker
                foreach column $columns {
                    set ${table}($column) {}
                }
            }
        }
        if {[info exists instances(identifier)]} {
            set new instances
            if {![info exists entries(instance)]} {set old entries}
            set ($this,oldFormat) 0
        } else {
            set old instances
            if {[info exists entries(instance)]} {set new entries}
            set ($this,oldFormat) 1
        }
        if {[info exists old] && [info exists new]} {
            set ($this,error) "database fatal error: \"$new\" table in new format but \"$old\" table in old format (see upgrading section in database documentation)"
        }
        foreach {data history} [64BitRows $this] {}
        if {$data != $history} {
            set ($this,error) "database fatal error: data and history tables have mismatched types for the row column"
        }
        set ($this,64Bits) $data
    }

    proc 64BitRows {this} {                               ;# return data and history table 64 bit rows support in a list of booleans
        if {$($this,odbc)} {
            if {$($this,type) eq "oracle"} {
                return [list 1 1]                                                 ;# base INTEGER type already supports huge numbers
            }
            if {$::tcl_platform(platform) eq "windows"} {                                                       ;### no checking ###
                return [list 1 1]  ;### tclodbc windows bug, sample bogus data found below: "{} 4 {} {} {} NO {} {} {} {} {} {}" ###

            }
            foreach data [concat [odbcConnection $this columns data] [odbcConnection $this columns history]] {
                foreach {qualifier owner name column typeCode type precision length scale radix nullable remarks} $data {}
                if {$column eq "row"} {
                    lappend list [string equal $type bigint]
                    continue
                }
            }
            return $list
        } else {
            foreach table [list data history] {
                set type [lindex [mysqlSelect $this "SHOW COLUMNS FROM $table LIKE 'row'" -flatlist] 1]
                lappend list [string match -nocase BIGINT* $type]
            }
            return $list
        }
    }

    # return date and time in a format suitable for direct insertion into a date and time column, or use in a query
    proc dateTime {this {seconds {}}} {
        if {$seconds eq ""} {set seconds [clock seconds]}                                       ;# use current time if not specified
        if {$($this,file)} {
            return $seconds                                            ;# SQLite cannot handle dates and times, just numbers or text
        }
        set string [clock format $seconds -format $(dateTimeFormat)]
        if {$($this,odbc) && ($($this,type) eq "oracle")} {                                                   ;# Oracle special case
            return "TO_DATE('$string', 'YYYY-MM-DD HH:MI:SS')"
        } else {
            return '$string'                                                                            ;# pre-formatted with quotes
        }
    }

}

if {$global::withGUI} { ;# >8

class database {

    proc modules {this} {
        variable sampleModule

        # ignore zero instance (may be needed in a converted old format table) and post creation instances
        if {$($this,odbc)} {
            set prefix $($this,prefix)
            set query "SELECT DISTINCT ${prefix}module FROM instances WHERE (${prefix}number > 0) AND (${prefix}start <= $($this,start)) ORDER BY ${prefix}module"
            set list [join [odbcConnection $this $query]]
        } else {
            set query "SELECT DISTINCT module FROM instances WHERE (number > 0) AND (start <= $($this,start)) ORDER BY module"
            if {$($this,file)} {
                set list [sqliteEvaluate $this $query]
            } else {
                set list [mysqlSelect $this $query -flatlist]
            }
        }
        if {$(includeSamples)} {
            foreach {instance module} [array get sampleModule] {lappend list $module}
            set list [lsort -dictionary $list]
        }
        return $list
    }

    proc moduleRange {this module {busyWidgets .}} {                              ;# return time boundaries for all module instances
        variable sampleModule

        if {$(includeSamples)} {
            foreach {instance name} [array get sampleModule] {
                if {$name eq $module} {return [datasets::range $name]}                                   ;# found simulated instance
            }
        }
        lifoLabel::push $global::messenger [mc {retrieving module instances range from database...}]
        busy 1 $busyWidgets
        # find time boundaries from all cell histories for that module instances:
        if {$($this,odbc)} {
            set prefix $($this,prefix)
            set query "SELECT MIN(${prefix}instant), MAX(${prefix}instant) FROM instances, history WHERE (${prefix}module = '$module') AND (${prefix}instance = ${prefix}number) AND (${prefix}instant <= $($this,start))"
            set list [join [odbcConnection $this $query]]
            if {$($this,type) eq "mssql"} {
                foreach {minimum maximum} $list {}
                set list [list [regsub {\..*$} $minimum {}] [regsub {\..*$} $maximum {}]]                      ;# prune milliseconds
            }
        } else {
            set query "SELECT MIN(instant), MAX(instant) FROM instances, history WHERE (module = '$module') AND (instance = number) AND (instant <= $($this,start))"
            if {$($this,file)} {
                set list [sqliteEvaluate $this $query]
                foreach {minimum maximum} $list {}
                if {$minimum ne ""} {                                 ;# if no history, a list of empty values is returned by SQLite
                    set list\
                        [list [clock format $minimum -format $(dateTimeFormat)] [clock format $maximum -format $(dateTimeFormat)]]
                }
            } else {
                set list [mysqlSelect $this $query -flatlist]
            }
        }
        busy 0 $busyWidgets
        lifoLabel::pop $global::messenger
        return $list
    }

    proc instances {this module} {
        variable sampleModule

        if {$(includeSamples)} {
            foreach {instance name} [array get sampleModule] {
                if {$name eq $module} {return $instance}                                                 ;# found simulated instance
            }
        }
        # ignore zero instance (may be needed in a converted old format table) and post creation instances
        if {$($this,odbc)} {
            set prefix $($this,prefix)
            set query "SELECT ${prefix}number FROM instances WHERE (${prefix}number > 0) AND (${prefix}module = '$module') AND (${prefix}start <= $($this,start)) ORDER BY ${prefix}number"
            return [join [odbcConnection $this $query]]
        } else {
            set query "SELECT number FROM instances WHERE (number > 0) AND (module = '$module') AND (start <= $($this,start)) ORDER BY number"
            if {$($this,file)} {
                return [sqliteEvaluate $this $query]
            } else {
                return [mysqlSelect $this $query -flatlist]
            }
        }
    }

    proc instanceRange {this instance {busyWidgets .}} {                                      ;# return time boundaries for instance
        variable sampleModule

        if {$(includeSamples) && [info exists sampleModule($instance)]} {
            return [datasets::range $sampleModule($instance)]
        }
        lifoLabel::push $global::messenger [mc {retrieving module instance range from database...}]
        busy 1 $busyWidgets
        # find time boundaries from all cell histories for that instance:
        if {$($this,odbc)} {
            set prefix $($this,prefix)
            set query "SELECT MIN(${prefix}instant), MAX(${prefix}instant) FROM history WHERE (${prefix}instance = $instance) AND (${prefix}instant <= $($this,start))"
            set list [join [odbcConnection $this $query]]
            if {$($this,type) eq "mssql"} {
                foreach {minimum maximum} $list {}
                set list [list [regsub {\..*$} $minimum {}] [regsub {\..*$} $maximum {}]]                      ;# prune milliseconds
            }
        } else {
            set query "SELECT MIN(instant), MAX(instant) FROM history WHERE (instance = $instance) AND (instant <= $($this,start))"
            if {$($this,file)} {
                set list [sqliteEvaluate $this $query]
                foreach {minimum maximum} $list {}
                if {$minimum ne ""} {                                 ;# if no history, a list of empty values is returned by SQLite
                    set list\
                        [list [clock format $minimum -format $(dateTimeFormat)] [clock format $maximum -format $(dateTimeFormat)]]
                }
            } else {
                set list [mysqlSelect $this $query -flatlist]
            }
        }
        busy 0 $busyWidgets
        lifoLabel::pop $global::messenger
        return $list
    }

    proc arguments {this instance} {
        variable sampleModule

        if {$(includeSamples) && [info exists sampleModule($instance)]} {
            set module $sampleModule($instance)
            return [list --description \"[set datasets::${module}(description)]\"]
        }
        if {$($this,odbc)} {
            set prefix $($this,prefix)
            set query "SELECT ${prefix}name, ${prefix}value FROM options WHERE ${prefix}instance = $instance ORDER BY ${prefix}name"
            return [join [odbcConnection $this $query]]
        } else {
            set query "SELECT name, value FROM options WHERE instance = $instance ORDER BY name"
            if {$($this,file)} {
                return [sqliteEvaluate $this $query]
            } else {
                return [mysqlSelect $this $query -flatlist]
            }
        }
    }

    proc identifier {this instance} {
        variable sampleModule

        if {$(includeSamples) && [info exists sampleModule($instance)]} {
            return $sampleModule($instance)<0>                                                    ;# there always is a sole instance
        }
        if {!$($this,oldFormat)} {                                                                            ;# new database format
            if {$($this,odbc)} {
                set prefix $($this,prefix)
                set query "SELECT ${prefix}identifier FROM instances WHERE ${prefix}number = $instance"
                set identifier [lindex [join [odbcConnection $this $query]] 0]
            } else {
                set query "SELECT identifier FROM instances WHERE number = $instance"
                if {$($this,file)} {
                    set identifier [lindex [sqliteEvaluate $this $query] 0]
                } else {
                    set identifier [lindex [mysqlSelect $this $query -flatlist] 0]
                }
            }
            if {$($this,error) ne ""} {return {}}                                                          ;# abort in case of error
            if {$identifier ne ""} {                                        ;# valid identifier was found in the new database format
                return $identifier
            }
        }           ;# else identifier column may have been added to the instances table in an old format database, so return module
        # note: old database format was never used against oracle and therefore no table column name prefixing is needed
        set query "SELECT module FROM instances WHERE number = $instance"
        if {$($this,odbc)} {
            return [lindex [join [odbcConnection $this $query]] 0]
        } else {
            return [lindex [mysqlSelect $this $query -flatlist] 0]
        }
    }

    proc version {this instance} {                                                           ;# return major.minor (2.1 for example)
        variable sampleModule

        if {$(includeSamples) && [info exists sampleModule($instance)]} {
            return [package present datasets]                                                                 ;# use package version
        }
        if {$($this,odbc)} {
            set prefix $($this,prefix)
            set query "SELECT ${prefix}major, ${prefix}minor FROM instances WHERE ${prefix}number = $instance"
            return [join [join [odbcConnection $this $query]] .]
        } else {
            set query "SELECT major, minor FROM instances WHERE number = $instance"
            if {$($this,file)} {
                return [join [sqliteEvaluate $this $query] .]
            } else {
                return [join [mysqlSelect $this $query -flatlist] .]
            }
        }
    }

    proc cellsData {this instance} {
        variable sampleModule

        if {$(includeSamples) && [info exists sampleModule($instance)]} {
            return [list 0 0 value {}]                                                                       ;# always a single cell
        }
        set list {}
        if {$($this,odbc)} {
            set prefix $($this,prefix)
            set query "SELECT ${prefix}row, ${prefix}entry, ${prefix}label, ${prefix}comment FROM data WHERE ${prefix}instance = $instance ORDER BY ${prefix}row, ${prefix}entry"
            foreach {row entry label comment} [join [odbcConnection $this $query]] {
                if {$row < 0} {set row [unsigned $this $row]}
                lappend list $row $entry $label $comment
            }
        } else {
            set query "SELECT row, entry, label, comment FROM data WHERE instance = $instance ORDER BY row, entry"
            if {$($this,file)} {
                foreach {row entry label comment} [sqliteEvaluate $this $query] {
                    if {$row < 0} {set row [unsigned $this $row]}
                    lappend list $row $entry $label $comment
                }
            } else {
                foreach {row entry label comment} [mysqlSelect $this $query -flatlist] {
                    if {$row < 0} {set row [unsigned $this $row]}
                    lappend list $row $entry $label $comment
                }
            }
        }
        return $list
    }

    # If time limits are not specified, return timestamp limits for whole data cell history.
    # If start and end times are specified, return timestamp limits for data cell history lying between specified parameter times.
    proc cellRange {this instance row entry {startSeconds {}} {endSeconds {}} {busyWidgets .}} {          ;# seconds as in UNIX time
        variable sampleModule
        variable ${this}cache

        if {$(includeSamples) && [info exists sampleModule($instance)]} {
            return [datasets::range $sampleModule($instance) $startSeconds $endSeconds]               ;# there is only a single cell
        }
        set index range,$instance,$row,$entry,$startSeconds,$endSeconds
        if {[info exists ${this}cache($index)]} {
            return [set ${this}cache($index)]
        }
        set odbc $($this,odbc)
        if {$odbc} {
            set prefix $($this,prefix)
            set query "SELECT MIN(${prefix}instant), MAX(${prefix}instant) FROM history WHERE (${prefix}instance = $instance) AND (${prefix}row = [signed $this $row]) AND (${prefix}entry = $entry) AND (${prefix}instant <= $($this,start))"
        } else {
            set query "SELECT MIN(instant), MAX(instant) FROM history WHERE (instance = $instance) AND (row = [signed $this $row]) AND (entry = $entry) AND (instant <= $($this,start))"
        }
        if {($startSeconds ne "") && ($endSeconds ne "")} {
            if {$odbc} {
                append query " AND (${prefix}instant BETWEEN [dateTime $this $startSeconds] AND [dateTime $this $endSeconds])"
            } else {
                append query " AND (instant BETWEEN [dateTime $this $startSeconds] AND [dateTime $this $endSeconds])"
            }
        }
        lifoLabel::push $global::messenger [mc {retrieving cell range from database...}]
        busy 1 $busyWidgets
        if {$($this,file)} {
            set list [sqliteEvaluate $this $query]
            foreach {minimum maximum} $list {}
            if {$minimum ne ""} {                                     ;# if no history, a list of empty values is returned by SQLite
                set list [list [clock format $minimum -format $(dateTimeFormat)] [clock format $maximum -format $(dateTimeFormat)]]
            }
        } elseif {$odbc} {
            set list [join [odbcConnection $this $query]]
            if {$($this,type) eq "mssql"} {
                foreach {minimum maximum} $list {}
                set list [list [regsub {\..*$} $minimum {}] [regsub {\..*$} $maximum {}]]                      ;# prune milliseconds
            }
        } else {
            set list [mysqlSelect $this $query -flatlist]
        }
        busy 0 $busyWidgets
        lifoLabel::pop $global::messenger
        return [set ${this}cache($index) $list]                                    ;# (a list of 2 empty strings if no history data)
    }

    proc moduleData {this instance} {
        variable sampleModule

        if {$(includeSamples) && [info exists sampleModule($instance)]} {
            set module $sampleModule($instance)
            return [list [list 0 1 value real [set datasets::${module}(description)] {}]]           ;# there is only a single column
        }
        if {!$($this,oldFormat)} {                                                   ;# new database format (starting at moomps 2.3)
            if {$($this,odbc)} {
                set prefix $($this,prefix)
                set query "SELECT ${prefix}number, ${prefix}indexed, ${prefix}label, ${prefix}type, ${prefix}message, ${prefix}anchor FROM entries WHERE ${prefix}instance = $instance"
                set list [odbcConnection $this $query]
            } else {
                set query "SELECT number, indexed, label, type, message, anchor FROM entries WHERE instance = $instance"
                if {$($this,file)} {
                    set list {}
                    foreach {number indexed label type message anchor} [sqliteEvaluate $this $query] {
                        lappend list [list $number $indexed $label $type $message $anchor]
                    }
                } else {
                    set list [mysqlSelect $this $query -list]
                }
            }
            if {$($this,error) ne ""} {return {}}                                                          ;# abort in case of error
            if {[llength $list] > 0} {                                            ;# valid data was found in the new database format
                return $list                                                                                        ;# list of lists
            }
        }        ;# else instance column may have been added to the entries table in an old format database, so try old method below
        # note: old database format was never used against oracle and therefore no table column name prefixing is needed
        set query "SELECT module, major, minor FROM instances WHERE number = $instance"
        if {$($this,odbc)} {
            foreach {module major minor} [join [odbcConnection $this $query]] {}
        } else {
            foreach {module major minor} [mysqlSelect $this $query -flatlist] {}
        }
        set query "SELECT number, indexed, label, type, message, anchor FROM entries WHERE (module = '$module') AND (major = $major) AND (minor = $minor)"
        if {$($this,odbc)} {
            return [odbcConnection $this $query]                                                                    ;# list of lists
        } else {
            return [mysqlSelect $this $query -list]                                                                 ;# list of lists
        }
    }

    # Return timestamp, value, timestamp, value, ... flat list.
    # If time limits are not specified, return all samples ever recorded for data cell.
    # If only end time is specified, return sole sample that recorded nearest (before or at) the specified time.
    # If start and end times are specified, return all samples recorded between (inclusive) specified times.
    proc cellHistory {this instance row entry startSeconds endSeconds last {busyWidgets .}} {             ;# seconds as in UNIX time
        variable ${this}cache
        variable sampleModule

        if {$(includeSamples) && [info exists sampleModule($instance)]} {
            return [datasets::history $sampleModule($instance) $startSeconds $endSeconds $last]       ;# there is only a single cell
        }
        set index history,$instance,$row,$entry,$startSeconds,$endSeconds
        if {$last && [info exists ${this}cache($index)]} {
            return [set ${this}cache($index)]
        }
        set odbc $($this,odbc)
        if {$odbc} {
            set prefix $($this,prefix)
            set query "SELECT "
            if {$($this,type) eq "mssql"} {          ;# prune milliseconds by converting to string in ODBC canonical date time style
                append query "CONVERT(CHAR(19), ${prefix}instant, 120)"
            } else {
                append query "${prefix}instant"
            }
            append query ", ${prefix}value FROM history WHERE (${prefix}instance = $instance) AND (${prefix}row = [signed $this $row]) AND (${prefix}entry = $entry) AND (${prefix}instant BETWEEN [dateTime $this $startSeconds] AND [dateTime $this $endSeconds]) AND (${prefix}instant <= $($this,start))"
        } else {
            set query "SELECT instant, value FROM history WHERE (instance = $instance) AND (row = [signed $this $row]) AND (entry = $entry) AND (instant BETWEEN [dateTime $this $startSeconds] AND [dateTime $this $endSeconds]) AND (instant <= $($this,start))"
        }
        if {$last} {                                                                                                  ;# sole sample
            if {$odbc} {
                switch $($this,type) {
                    oracle {
                        append query " AND (ROWNUM <= 1) ORDER BY ${prefix}instant DESC"
                    }
                    mssql {
                        regsub {^SELECT} $query {SELECT TOP 1} query                                          ;# LIMIT functionality
                        append query " ORDER BY ${prefix}instant DESC"
                    }
                    default {
                        append query " ORDER BY ${prefix}instant DESC "
                        regsub -all %l $($this,limit) 1 limit
                        append query $limit
                    }
                }
            } else {
                append query " ORDER BY instant DESC LIMIT 1"
            }
        } else {                                                                                                            ;# range
            if {$odbc} {
                append query " ORDER BY ${prefix}instant"
            } else {
                append query " ORDER BY instant"
            }
        }
        if {$last} {                                                                                                  ;# sole sample
            lifoLabel::push $global::messenger [mc {retrieving cell value before end cursor from database...}]
        } else {                                                                                                            ;# range
            lifoLabel::push $global::messenger [mc {retrieving cell history from database...}]
        }
        busy 1 $busyWidgets
        if {$($this,file)} {
            set list {}
            foreach {seconds value} [sqliteEvaluate $this $query] {
                lappend list [clock format $seconds -format $(dateTimeFormat)] $value
            }
        } elseif {$odbc} {
            set list [join [odbcConnection $this $query]]
        } else {                                                                    ;# MySQL: format time stamp into a standard form
            set list [mysqlSelect $this $query -flatlist]
        }
        busy 0 $busyWidgets
        lifoLabel::pop $global::messenger
        if {$last} {                                                ;# always cache single values since it does not take much memory
            set ${this}cache($index) $list
        }
        return $list                                                                              ;# empty in case of database error
    }

    # return SQL query in strings list form, so it can be used (for example by pasting in a client) to retrieve data cell history
    proc historyQuery {this instance row entry start end} {
        variable sampleModule

        if {$(includeSamples) && [info exists sampleModule($instance)]} {
            return [list {} {} {}]                                                                                ;# not in database
        }
        set odbc $($this,odbc)
        if {$odbc} {
            set prefix $($this,prefix)
            lappend list \
"SELECT ${prefix}instant, ${prefix}value FROM history
    WHERE (${prefix}instance = $instance) AND (${prefix}row = [signed $this $row]) AND (${prefix}entry = $entry)"
        } else {
            lappend list \
"SELECT instant, value FROM history
    WHERE (instance = $instance) AND (row = [signed $this $row]) AND (entry = $entry)"
        }
        if {($start ne "") && ($end ne "")} {                                                                        ;# else no data
            if {$odbc} {
                if {$($this,type) eq "oracle"} {                                                              ;# Oracle special case
                    lappend list "AND (${prefix}instant >= TO_DATE('$start', 'YYYY-MM-DD HH:MI:SS')) AND (${prefix}instant <= TO_DATE('$end', 'YYYY-MM-DD HH:MI:SS'))"
                } else {
                    lappend list "AND (${prefix}instant >= '$start') AND (${prefix}instant <= '$end')"
                }
            } else {
                lappend list "AND (instant >= '$start') AND (instant <= '$end')"
            }
        } else {
            lappend list {}                                                                              ;# time limits are optional
        }
        if {$odbc} {
            lappend list "ORDER BY ${prefix}instant"
        } else {
            lappend list "ORDER BY instant"
        }
        return $list
    }

}

} ;# >8

class database {

    # the code below is executed in the main interpreter, since there is only 1 database object, which corresponds to 1 connection

    proc register {this instanceData} {                                                                  ;# array contents as a list
        array set data $instanceData
        set module $data(module)
        set file $($this,file)
        set odbc $($this,odbc)
        foreach {major minor} [lrange [split $data(version) .] 0 1] {}                    ;# only keep major and minor version parts
        if {$odbc} {
            set prefix $($this,prefix)
            set arguments {}
        }
        # first find out whether there is a recorded instance for the module with the same major version number and options
        if {![info exists data(options)] || ([llength $data(options)] == 0)} {                            ;# no options special case
            if {$odbc} {
                set query "SELECT ${prefix}number FROM instances LEFT OUTER JOIN options ON ${prefix}number = ${prefix}instance WHERE (${prefix}module = '$module') AND (${prefix}major = $major) GROUP BY ${prefix}number HAVING COUNT(${prefix}instance) = 0"
            } else {
                set query "SELECT number FROM instances LEFT OUTER JOIN options ON number = instance WHERE (module = '$module') AND (major = $major) GROUP BY number HAVING COUNT(instance) = 0"
            }
        } else {
            # use a left join on options table self, the matching instance being the unique one if any, with the correct number of
            # options and that all match
            if {$odbc} {
                set query "SELECT ${prefix}number FROM instances, options LEFT OUTER JOIN options AS joined ON (options.${prefix}instance = joined.${prefix}instance) AND (options.${prefix}name = joined.${prefix}name)"
            } else {
                set query "SELECT number FROM instances, options LEFT OUTER JOIN options AS joined ON (options.instance = joined.instance) AND (options.name = joined.name)"
            }
            set count 0
            foreach {name value} $data(options) {                                   ;# options are switch/value pairs in a flat list
                if {$count == 0} {
                    append query " AND ("
                } else {
                    append query " OR "
                }
                if {$odbc} {      ;# pass names (which may contain special characters) as statement arguments for automatic escaping
                    append query "((options.${prefix}name = "
                    append query ?
                    lappend arguments $name
                    append query ") AND (options.${prefix}value "
                } else {
                    append query "((options.name = "
                    if {$file} {
                        append query '[sqliteEscape $name]'                                    ;# possibly escape special characters
                    } else {
                        append query '[mysqlescape $name]'                                     ;# possibly escape special characters
                    }
                    append query ") AND (options.value "
                }
                if {$value eq ""} {
                    append query "IS NULL"
                } elseif {[regexp $global::passwordOptionExpression $name]} {
                    append query "LIKE '%'"     ;# any string as password in not stored in clear and may change in instance lifetime
                } else {
                    append query "= "
                    if {$file} {
                        append query '[sqliteEscape $value]'                                   ;# possibly escape special characters
                    } elseif {$odbc} {
                        # pass values (which may contain special characters) as statement arguments for automatic escaping
                        append query ?
                        lappend arguments $value
                    } else {
                        append query '[mysqlescape $value]'                                    ;# possibly escape special characters
                    }
                }
                append query "))"
                incr count
            }
            if {$count > 0} {                                                                                  ;# there were options
                append query ")"
            }
            if {$odbc} {                           ;# note: grouping by options instance is not necessary but required by PostgreSQL
                append query " WHERE (${prefix}module = '$module') AND (${prefix}major = $major) AND (${prefix}number = options.${prefix}instance) GROUP BY ${prefix}number, options.${prefix}instance HAVING (COUNT(*) = COUNT(joined.${prefix}instance)) AND (COUNT(*) = $count)"
            } else {
                append query " WHERE (module = '$module') AND (major = $major) AND (number = options.instance) GROUP BY number, options.instance HAVING (COUNT(*) = COUNT(joined.instance)) AND (COUNT(*) = $count)"
            }
        }
        if {$file} {
            set instance [lindex [sqliteEvaluate $this $query] 0]
        } elseif {$odbc} {
            set instance [lindex [join [odbcConnection $this $query $arguments]] 0]
        } else {
            set instance [lindex [mysqlSelect $this $query -flatlist] 0]
        }
        if {$($this,error) ne ""} {return {}}                                                              ;# abort in case of error
        if {$instance eq ""} {                  ;# no matching instance: create new entries for that module and its specific options
            if {[info exists data(options)]} {set options $data(options)} else {set options {}}
            set instance [insertInstance $this $module $data(identifier) $major $minor $options]
        } else {
            updateInstance $this $instance $data(identifier) $minor
        }
        if {$($this,error) ne ""} {return {}}                                                              ;# abort in case of error
        updateEntries $this $instance $data(indexColumns) $data(data)                                       ;# always update entries
        if {$($this,error) ne ""} {return {}}                                                              ;# abort in case of error
        return $instance
    }

    proc insertInstance {this module identifier major minor options} {
        set file $($this,file)
        set odbc $($this,odbc)
        if {[info exists ($this,lock)]} {                             ;# possibly lock table to prevent concurrency induced problems
            regsub -all %t $($this,lock) instances statement
            if {$odbc} {odbcConnection $this $statement} else {mysqlExecute $this $statement}
        } elseif {$odbc} {       ;# handle transaction via ODBC layer (executing the corresponding database command not recommended)
            odbcConnection $this set autocommit off
        }
        if {$odbc} {
            set prefix $($this,prefix)
            set query "SELECT MAX(${prefix}number) FROM instances"
            set instance [lindex [join [odbcConnection $this $query]] 0]
        } else {
            set query "SELECT MAX(number) FROM instances"
            if {$file} {
                set instance [lindex [sqliteEvaluate $this $query] 0]
            } else {
                set instance [lindex [mysqlSelect $this $query -flatlist] 0]
            }
        }
        if {$($this,error) ne ""} {return {}}                                                              ;# abort in case of error
        if {$instance eq ""} {set instance 0}                                                         ;# there were no instances yet
        incr instance                                                                                            ;# take next number
        set statement "INSERT INTO instances VALUES ($instance, [dateTime $this], "       ;# store very first instance creation time
        if {$file} {
            append statement "'[sqliteEscape $module]', '[sqliteEscape $identifier]', "                 ;# escape special characters
        } elseif {$odbc} {        ;# pass names (which may contain special characters) as statement arguments for automatic escaping
            append statement {?, ?, }
            set arguments [list $module $identifier]
        } else {
            append statement "'[mysqlescape $module]', '[mysqlescape $identifier]', "                   ;# escape special characters
        }
        append statement "$major, $minor)"                                                ;# store very first instance minor version
        if {$file} {
            sqliteEvaluate $this $statement
        } elseif {$odbc} {
            odbcConnection $this $statement $arguments
        } else {
            mysqlExecute $this $statement
        }
        if {$($this,error) ne ""} {return {}}                                                              ;# abort in case of error
        if {[info exists ($this,unlock)]} {
            regsub -all %t $($this,unlock) instances statement
            if {$odbc} {odbcConnection $this $statement} else {mysqlExecute $this $statement}
        } elseif {$odbc} {
            odbcConnection $this commit                                                         ;# handle transaction via ODBC layer
            odbcConnection $this set autocommit on
        }
        foreach {name value} $options {
            if {$odbc} {
                set arguments {}
            }
            set statement "INSERT INTO options VALUES ($instance, "
            if {$file} {
                append statement '[sqliteEscape $name]'                                        ;# possibly escape special characters
            } elseif {$odbc} {    ;# pass names (which may contain special characters) as statement arguments for automatic escaping
                append statement ?
                lappend arguments $name
            } else {
                append statement '[mysqlescape $name]'                                         ;# possibly escape special characters
            }
            append statement ", "
            if {$value eq ""} {
                append statement NULL
            } else {
                if {[regexp $global::passwordOptionExpression $name]} {
                    set value *                                                              ;# do not store readable password value
                }
                if {$file} {
                    append statement '[sqliteEscape $value]'                                   ;# possibly escape special characters
                } elseif {$odbc} {
                    # pass names (which may contain special characters) as statement arguments for automatic escaping
                    append statement ?
                    lappend arguments $value
                } else {
                    append statement '[mysqlescape $value]'                                    ;# possibly escape special characters
                }
            }
            append statement )
            if {$file} {
                sqliteEvaluate $this $statement
            } elseif {$odbc} {
                odbcConnection $this $statement $arguments
            } else {
                mysqlExecute $this $statement
            }
            if {$($this,error) ne ""} {return {}}                                                          ;# abort in case of error
        }
        return $instance
    }

    proc updateInstance {this instance identifier minor} {                                         ;# update instance variable parts
        set odbc $($this,odbc)
        set file $($this,file)
        if {$odbc} {                ;# pass name (which may contain special characters) as statement argument for automatic escaping
            set prefix $($this,prefix)
            set statement "UPDATE instances SET ${prefix}start = [dateTime $this], ${prefix}identifier = "
            append statement ?
            append statement ", ${prefix}minor = $minor WHERE ${prefix}number = $instance"
        } else {
            set statement "UPDATE instances SET start = [dateTime $this], identifier = "
            if {$file} {
                append statement '[sqliteEscape $identifier]'                                           ;# escape special characters
            } else {
                append statement '[mysqlescape $identifier]'                                            ;# escape special characters
            }
            append statement ", minor = $minor WHERE number = $instance"
        }
        if {$file} {
            sqliteEvaluate $this $statement
        } elseif {$odbc} {
            odbcConnection $this $statement [list $identifier]
        } else {
            mysqlExecute $this $statement
        }
    }

    proc updateEntries {this instance indexColumns data} {
        set file $($this,file)
        set odbc $($this,odbc)
        # delete existing entries if any first for resiliency sake
        if {$odbc} {
            set prefix $($this,prefix)
            set statement "DELETE FROM entries WHERE ${prefix}instance = $instance"
            odbcConnection $this $statement
        } else {
            set statement "DELETE FROM entries WHERE instance = $instance"
            if {$file} {
                sqliteEvaluate $this $statement
            } else {
                mysqlExecute $this $statement
            }
        }
        if {$($this,error) ne ""} return                                                                   ;# abort in case of error
        foreach index $indexColumns {set indexed($index) {}}
        set index 0                                                                                                  ;# column index
        foreach {label type message anchor} $data {                                                 ;# now store module columns data
            if {$odbc} {
                set arguments {}
            }
            # entries table may still include module, major and minor columns, so use a statement with named columns:
            if {$odbc} {
                set statement "INSERT INTO entries (${prefix}instance, ${prefix}number, ${prefix}indexed, ${prefix}label, ${prefix}type, ${prefix}message, ${prefix}anchor) VALUES ("
            } else {
                set statement "INSERT INTO entries (instance, number, indexed, label, type, message, anchor) VALUES ("
            }
            append statement "$instance, $index, [info exists indexed($index)], "
            if {$file} {
                append statement '[sqliteEscape $label]'                                       ;# possibly escape special characters
            } elseif {$odbc} {   ;# pass labels (which may contain special characters) as statement arguments for automatic escaping
                append statement ?
                lappend arguments $label
            } else {
                append statement '[mysqlescape $label]'                                        ;# possibly escape special characters
            }
            append statement ", '$type', "
            if {$file} {
                append statement '[sqliteEscape $message]'
            } elseif {$odbc} {
                append statement ?
                lappend arguments $message
            } else {
                append statement '[mysqlescape $message]'
            }
            append statement ", "
            if {$anchor eq ""} {
                append statement NULL
            } else {
                append statement '$anchor'
            }
            append statement )
            if {$file} {
                sqliteEvaluate $this $statement
            } elseif {$odbc} {
                odbcConnection $this $statement $arguments
            } else {
                mysqlExecute $this $statement
            }
            if {$($this,error) ne ""} return                                                               ;# abort in case of error
            incr index
        }
    }

    proc update {this instance row entry value} {                                                          ;# update data cell value
        set file $($this,file)
        set odbc $($this,odbc)
        if {![info exists ($this,connection)]} {                                ;# disconnected on prior error: attempt to reconnect
            if {$file} {
                sqliteOpen $this
            } elseif {$odbc} {
                odbcConnect $this
            } else {
                mysqlConnect $this
            }
            if {$($this,error) ne ""} return                                                                          ;# retry later
        }
        if {$odbc} {
            set arguments {}
        }
        set statement "INSERT"
        if {!$file && (!$odbc || ($($this,type) eq "mysql"))} {                                                    ;# MySQL database
            # make statement very quick so as not to affect modules, such as snmp which may time out if MySQL server is very busy
            append statement " DELAYED"
        }
        append statement " INTO history VALUES ([dateTime $this], $instance, [signed $this $row], $entry, "
        if {$value eq "?"} {                                                                                        ;# unknown value
            append statement NULL
        } elseif {$odbc} {
            append statement ?
            lappend arguments $value
        } else {
            if {$file} {
                append statement '[sqliteEscape $value]'                                       ;# possibly escape special characters
            } else {
                append statement '[mysqlescape $value]'                                        ;# possibly escape special characters
            }
        }
        append statement )
        if {$file} {
            sqliteEvaluate $this $statement
        } elseif {$odbc} {
            odbcConnection $this $statement $arguments
        } else {
            mysqlExecute $this $statement
        }
        if {$($this,error) ne ""} {                                                               ;# error: probably a disconnection
            disconnect $this                                                                      ;# just in case, but ignore errors
        }
    }

    proc monitor {this instance row entry label comment} {                                  ;# initialize some data cell static data
        set file $($this,file)
        set odbc $($this,odbc)
        if {$odbc} {
            set arguments {}
        }
        if {$odbc} {
            set prefix $($this,prefix)
            set statement "DELETE FROM data WHERE (${prefix}instance = $instance) AND (${prefix}row = [signed $this $row]) AND (${prefix}entry = $entry)"
            odbcConnection $this $statement
        } else {
            set statement "DELETE FROM data WHERE (instance = $instance) AND (row = [signed $this $row]) AND (entry = $entry)"
            if {$file} {
                sqliteEvaluate $this $statement
            } else {
                mysqlExecute $this $statement
            }
        }
        set statement "INSERT INTO data VALUES ($instance, [signed $this $row], $entry, "
        if {$file} {
            append statement '[sqliteEscape $label]'                                           ;# possibly escape special characters
        } elseif {$odbc} {        ;# pass label (which may contain special characters) as statement arguments for automatic escaping
            append statement ?
            lappend arguments $label
        } else {
            append statement '[mysqlescape $label]'                                            ;# possibly escape special characters
        }
        append statement ", "
        if {$comment eq ""} {
            append statement NULL
        } elseif {$odbc} {
            append statement ?
            lappend arguments $comment
        } else {
            if {$file} {
                append statement '[sqliteEscape $comment]'                                     ;# possibly escape special characters
            } else {
                append statement '[mysqlescape $comment]'                                      ;# possibly escape special characters
            }
        }
        append statement )
        if {$file} {
            sqliteEvaluate $this $statement
        } elseif {$odbc} {
            odbcConnection $this $statement $arguments
        } else {
            mysqlExecute $this $statement
        }
    }

    proc sqliteEscape {string} {
        regsub -all ' $string '' string                                                                             ;# escape quotes
        return $string
    }

    proc disconnect {this} {
        if {$($this,file)} {
            sqliteClose $this
        } elseif {$($this,odbc)} {
            odbcDisconnect $this
        } else {
            mysqlDisconnect $this
        }
    }

if {$tcl_platform(wordSize) == 8} {                                                                           ;# 64 bit Tcl platform
    proc signed {this integer} {                                  ;# if database cannot handle 64 bits, integer is 32 bits in length
        if {$($this,64Bits) || ($integer <= 0x7FFFFFFF)} {return [expr {$integer}]} else {return [expr {$integer - 0x100000000}]}
    }
} else {                                                                                                      ;# 32 bit Tcl platform
    proc signed {this integer} {
        if {$($this,64Bits)} {return [expr {$integer}]} else {return [expr {int($integer)}]}
    }
}
    proc unsigned {this integer} {
        if {$($this,64Bits)} {return [format %lu $integer]} else {return [expr {$integer & 0xFFFFFFFF}]}
    }

}
