blackroad-os/backroad
9
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP: sqlite support

Browse Source
This commit is contained in:
Prabhat Khera
2022-10-15 00:12:51 +13:00
parent 7a6ff10268
commit a304b9c7ef
102 changed files with 4450 additions and 13197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

434
api/database/boltdb/db.go
View File

@@ -1,434 +0,0 @@
package boltdb
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"time"
dserrors "github.com/portainer/portainer/api/dataservices/errors"
"github.com/rs/zerolog/log"
bolt "go.etcd.io/bbolt"
)
const (
DatabaseFileName = "portainer.db"
EncryptedDatabaseFileName = "portainer.edb"
)
var (
ErrHaveEncryptedAndUnencrypted = errors.New("Portainer has detected both an encrypted and un-encrypted database and cannot start. Only one database should exist")
ErrHaveEncryptedWithNoKey = errors.New("The portainer database is encrypted, but no secret was loaded")
)
type DbConnection struct {
Path string
MaxBatchSize int
MaxBatchDelay time.Duration
InitialMmapSize int
EncryptionKey []byte
isEncrypted bool
*bolt.DB
}
// GetDatabaseFileName get the database filename
func (connection *DbConnection) GetDatabaseFileName() string {
if connection.IsEncryptedStore() {
return EncryptedDatabaseFileName
}
return DatabaseFileName
}
// GetDataseFilePath get the path + filename for the database file
func (connection *DbConnection) GetDatabaseFilePath() string {
if connection.IsEncryptedStore() {
return path.Join(connection.Path, EncryptedDatabaseFileName)
}
return path.Join(connection.Path, DatabaseFileName)
}
// GetStorePath get the filename and path for the database file
func (connection *DbConnection) GetStorePath() string {
return connection.Path
}
func (connection *DbConnection) SetEncrypted(flag bool) {
connection.isEncrypted = flag
}
// Return true if the database is encrypted
func (connection *DbConnection) IsEncryptedStore() bool {
return connection.getEncryptionKey() != nil
}
// NeedsEncryptionMigration returns true if database encryption is enabled and
// we have an un-encrypted DB that requires migration to an encrypted DB
func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
// Cases: Note, we need to check both portainer.db and portainer.edb
// to determine if it's a new store. We only need to differentiate between cases 2,3 and 5
// 1) portainer.edb + key => False
// 2) portainer.edb + no key => ERROR Fatal!
// 3) portainer.db + key => True (needs migration)
// 4) portainer.db + no key => False
// 5) NoDB (new) + key => False
// 6) NoDB (new) + no key => False
// 7) portainer.db & portainer.edb => ERROR Fatal!
// If we have a loaded encryption key, always set encrypted
if connection.EncryptionKey != nil {
connection.SetEncrypted(true)
}
// Check for portainer.db
dbFile := path.Join(connection.Path, DatabaseFileName)
_, err := os.Stat(dbFile)
haveDbFile := err == nil
// Check for portainer.edb
edbFile := path.Join(connection.Path, EncryptedDatabaseFileName)
_, err = os.Stat(edbFile)
haveEdbFile := err == nil
if haveDbFile && haveEdbFile {
// 7 - encrypted and unencrypted db?
return false, ErrHaveEncryptedAndUnencrypted
}
if haveDbFile && connection.EncryptionKey != nil {
// 3 - needs migration
return true, nil
}
if haveEdbFile && connection.EncryptionKey == nil {
// 2 - encrypted db, but no key?
return false, ErrHaveEncryptedWithNoKey
}
// 1, 4, 5, 6
return false, nil
}
// Open opens and initializes the BoltDB database.
func (connection *DbConnection) Open() error {
log.Info().Str("filename", connection.GetDatabaseFileName()).Msg("loading PortainerDB")
// Now we open the db
databasePath := connection.GetDatabaseFilePath()
db, err := bolt.Open(databasePath, 0600, &bolt.Options{
Timeout: 1 * time.Second,
InitialMmapSize: connection.InitialMmapSize,
})
if err != nil {
return err
}
db.MaxBatchSize = connection.MaxBatchSize
db.MaxBatchDelay = connection.MaxBatchDelay
connection.DB = db
return nil
}
// Close closes the BoltDB database.
// Safe to being called multiple times.
func (connection *DbConnection) Close() error {
if connection.DB != nil {
return connection.DB.Close()
}
return nil
}
// BackupTo backs up db to a provided writer.
// It does hot backup and doesn't block other database reads and writes
func (connection *DbConnection) BackupTo(w io.Writer) error {
return connection.View(func(tx *bolt.Tx) error {
_, err := tx.WriteTo(w)
return err
})
}
func (connection *DbConnection) ExportRaw(filename string) error {
databasePath := connection.GetDatabaseFilePath()
if _, err := os.Stat(databasePath); err != nil {
return fmt.Errorf("stat on %s failed: %s", databasePath, err)
}
b, err := connection.ExportJSON(databasePath, true)
if err != nil {
return err
}
return ioutil.WriteFile(filename, b, 0600)
}
// ConvertToKey returns an 8-byte big endian representation of v.
// This function is typically used for encoding integer IDs to byte slices
// so that they can be used as BoltDB keys.
func (connection *DbConnection) ConvertToKey(v int) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(v))
return b
}
// CreateBucket is a generic function used to create a bucket inside a database database.
func (connection *DbConnection) SetServiceName(bucketName string) error {
return connection.Batch(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
return err
})
}
// GetObject is a generic function used to retrieve an unmarshalled object from a database database.
func (connection *DbConnection) GetObject(bucketName string, key []byte, object interface{}) error {
var data []byte
err := connection.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
value := bucket.Get(key)
if value == nil {
return dserrors.ErrObjectNotFound
}
data = make([]byte, len(value))
copy(data, value)
return nil
})
if err != nil {
return err
}
return connection.UnmarshalObjectWithJsoniter(data, object)
}
func (connection *DbConnection) getEncryptionKey() []byte {
if !connection.isEncrypted {
return nil
}
return connection.EncryptionKey
}
// UpdateObject is a generic function used to update an object inside a database database.
func (connection *DbConnection) UpdateObject(bucketName string, key []byte, object interface{}) error {
data, err := connection.MarshalObject(object)
if err != nil {
return err
}
return connection.Batch(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
return bucket.Put(key, data)
})
}
// DeleteObject is a generic function used to delete an object inside a database database.
func (connection *DbConnection) DeleteObject(bucketName string, key []byte) error {
return connection.Batch(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
return bucket.Delete(key)
})
}
// DeleteAllObjects delete all objects where matching() returns (id, ok).
// TODO: think about how to return the error inside (maybe change ok to type err, and use "notfound"?
func (connection *DbConnection) DeleteAllObjects(bucketName string, matching func(o interface{}) (id int, ok bool)) error {
return connection.Batch(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
var obj interface{}
err := connection.UnmarshalObject(v, &obj)
if err != nil {
return err
}
if id, ok := matching(obj); ok {
err := bucket.Delete(connection.ConvertToKey(id))
if err != nil {
return err
}
}
}
return nil
})
}
// GetNextIdentifier is a generic function that returns the specified bucket identifier incremented by 1.
func (connection *DbConnection) GetNextIdentifier(bucketName string) int {
var identifier int
connection.Batch(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
id, err := bucket.NextSequence()
if err != nil {
return err
}
identifier = int(id)
return nil
})
return identifier
}
// CreateObject creates a new object in the bucket, using the next bucket sequence id
func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64) (int, interface{})) error {
return connection.Batch(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
seqId, _ := bucket.NextSequence()
id, obj := fn(seqId)
data, err := connection.MarshalObject(obj)
if err != nil {
return err
}
return bucket.Put(connection.ConvertToKey(int(id)), data)
})
}
// CreateObjectWithId creates a new object in the bucket, using the specified id
func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, obj interface{}) error {
return connection.Batch(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
data, err := connection.MarshalObject(obj)
if err != nil {
return err
}
return bucket.Put(connection.ConvertToKey(id), data)
})
}
// CreateObjectWithStringId creates a new object in the bucket, using the specified id
func (connection *DbConnection) CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error {
return connection.Batch(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
data, err := connection.MarshalObject(obj)
if err != nil {
return err
}
return bucket.Put(id, data)
})
}
func (connection *DbConnection) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
err := connection.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
err := connection.UnmarshalObject(v, obj)
if err != nil {
return err
}
obj, err = append(obj)
if err != nil {
return err
}
}
return nil
})
return err
}
// TODO: decide which Unmarshal to use, and use one...
func (connection *DbConnection) GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
err := connection.View(func(tx *bolt.Tx) error {
bucket := tx.Bucket([]byte(bucketName))
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
err := connection.UnmarshalObjectWithJsoniter(v, obj)
if err != nil {
return err
}
obj, err = append(obj)
if err != nil {
return err
}
}
return nil
})
return err
}
func (connection *DbConnection) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error {
return connection.View(func(tx *bolt.Tx) error {
cursor := tx.Bucket([]byte(bucketName)).Cursor()
for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {
err := connection.UnmarshalObjectWithJsoniter(v, obj)
if err != nil {
return err
}
obj, err = append(obj)
if err != nil {
return err
}
}
return nil
})
}
// BackupMetadata will return a copy of the boltdb sequence numbers for all buckets.
func (connection *DbConnection) BackupMetadata() (map[string]interface{}, error) {
buckets := map[string]interface{}{}
err := connection.View(func(tx *bolt.Tx) error {
err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
bucketName := string(name)
bucket = tx.Bucket([]byte(bucketName))
seqId := bucket.Sequence()
buckets[bucketName] = int(seqId)
return nil
})
return err
})
return buckets, err
}
// RestoreMetadata will restore the boltdb sequence numbers for all buckets.
func (connection *DbConnection) RestoreMetadata(s map[string]interface{}) error {
var err error
for bucketName, v := range s {
id, ok := v.(float64) // JSON ints are unmarshalled to interface as float64. See: https://pkg.go.dev/encoding/json#Decoder.Decode
if !ok {
log.Error().Str("bucket", bucketName).Msg("failed to restore metadata to bucket, skipped")
continue
}
err = connection.Batch(func(tx *bolt.Tx) error {
bucket, err := tx.CreateBucketIfNotExists([]byte(bucketName))
if err != nil {
return err
}
return bucket.SetSequence(uint64(id))
})
}
return err
}

124
api/database/boltdb/db_test.go
View File

@@ -1,124 +0,0 @@
package boltdb
import (
"os"
"path"
"testing"
"github.com/stretchr/testify/assert"
)
func Test_NeedsEncryptionMigration(t *testing.T) {
// Test the specific scenarios mentioned in NeedsEncryptionMigration
// i.e.
// Cases: Note, we need to check both portainer.db and portainer.edb
// to determine if it's a new store. We only need to differentiate between cases 2,3 and 5
// 1) portainer.edb + key => False
// 2) portainer.edb + no key => ERROR Fatal!
// 3) portainer.db + key => True (needs migration)
// 4) portainer.db + no key => False
// 5) NoDB (new) + key => False
// 6) NoDB (new) + no key => False
// 7) portainer.db & portainer.edb (key not important) => ERROR Fatal!
is := assert.New(t)
dir := t.TempDir()
cases := []struct {
name string
dbname string
key bool
expectError error
expectResult bool
}{
{
name: "portainer.edb + key",
dbname: EncryptedDatabaseFileName,
key: true,
expectError: nil,
expectResult: false,
},
{
name: "portainer.db + key (migration needed)",
dbname: DatabaseFileName,
key: true,
expectError: nil,
expectResult: true,
},
{
name: "portainer.db + no key",
dbname: DatabaseFileName,
key: false,
expectError: nil,
expectResult: false,
},
{
name: "NoDB (new) + key",
dbname: "",
key: false,
expectError: nil,
expectResult: false,
},
{
name: "NoDB (new) + no key",
dbname: "",
key: false,
expectError: nil,
expectResult: false,
},
// error tests
{
name: "portainer.edb + no key",
dbname: EncryptedDatabaseFileName,
key: false,
expectError: ErrHaveEncryptedWithNoKey,
expectResult: false,
},
{
name: "portainer.db & portainer.edb",
dbname: "both",
key: true,
expectError: ErrHaveEncryptedAndUnencrypted,
expectResult: false,
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
connection := DbConnection{Path: dir}
if tc.dbname == "both" {
// Special case. If portainer.db and portainer.edb exist.
dbFile1 := path.Join(connection.Path, DatabaseFileName)
f, _ := os.Create(dbFile1)
f.Close()
defer os.Remove(dbFile1)
dbFile2 := path.Join(connection.Path, EncryptedDatabaseFileName)
f, _ = os.Create(dbFile2)
f.Close()
defer os.Remove(dbFile2)
} else if tc.dbname != "" {
dbFile := path.Join(connection.Path, tc.dbname)
f, _ := os.Create(dbFile)
f.Close()
defer os.Remove(dbFile)
}
if tc.key {
connection.EncryptionKey = []byte("secret")
}
result, err := connection.NeedsEncryptionMigration()
is.Equal(tc.expectError, err, "Fatal Error failure. Test: %s", tc.name)
is.Equal(result, tc.expectResult, "Failed test: %s", tc.name)
})
}
}

113
api/database/boltdb/export.go
View File

@@ -1,113 +0,0 @@
package boltdb
import (
"encoding/json"
"time"
"github.com/rs/zerolog/log"
bolt "go.etcd.io/bbolt"
)
func backupMetadata(connection *bolt.DB) (map[string]interface{}, error) {
buckets := map[string]interface{}{}
err := connection.View(func(tx *bolt.Tx) error {
err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
bucketName := string(name)
bucket = tx.Bucket([]byte(bucketName))
seqId := bucket.Sequence()
buckets[bucketName] = int(seqId)
return nil
})
return err
})
return buckets, err
}
// ExportJSON creates a JSON representation from a DbConnection. You can include
// the database's metadata or ignore it. Ensure the database is closed before
// using this function.
// inspired by github.com/konoui/boltdb-exporter (which has no license)
// but very much simplified, based on how we use boltdb
func (c *DbConnection) ExportJSON(databasePath string, metadata bool) ([]byte, error) {
log.Debug().Str("databasePath", databasePath).Msg("exportJson")
connection, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second, ReadOnly: true})
if err != nil {
return []byte("{}"), err
}
defer connection.Close()
backup := make(map[string]interface{})
if metadata {
meta, err := backupMetadata(connection)
if err != nil {
log.Error().Err(err).Msg("failed exporting metadata")
}
backup["__metadata"] = meta
}
err = connection.View(func(tx *bolt.Tx) error {
err = tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
bucketName := string(name)
var list []interface{}
version := make(map[string]string)
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
if v == nil {
continue
}
var obj interface{}
err := c.UnmarshalObject(v, &obj)
if err != nil {
log.Error().
Str("bucket", bucketName).
Str("object", string(v)).
Err(err).
Msg("failed to unmarshal")
obj = v
}
if bucketName == "version" {
version[string(k)] = string(v)
} else {
list = append(list, obj)
}
}
if bucketName == "version" {
backup[bucketName] = version
return nil
}
if len(list) > 0 {
if bucketName == "ssl" ||
bucketName == "settings" ||
bucketName == "tunnel_server" {
backup[bucketName] = nil
if len(list) > 0 {
backup[bucketName] = list[0]
}
return nil
}
backup[bucketName] = list
return nil
}
return nil
})
return err
})
if err != nil {
return []byte("{}"), err
}
return json.MarshalIndent(backup, "", " ")
}

133
api/database/boltdb/json.go
View File

@@ -1,133 +0,0 @@
package boltdb
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/json"
"fmt"
"io"
jsoniter "github.com/json-iterator/go"
"github.com/pkg/errors"
)
var errEncryptedStringTooShort = fmt.Errorf("encrypted string too short")
// MarshalObject encodes an object to binary format
func (connection *DbConnection) MarshalObject(object interface{}) (data []byte, err error) {
// Special case for the VERSION bucket. Here we're not using json
if v, ok := object.(string); ok {
data = []byte(v)
} else {
data, err = json.Marshal(object)
if err != nil {
return data, err
}
}
if connection.getEncryptionKey() == nil {
return data, nil
}
return encrypt(data, connection.getEncryptionKey())
}
// UnmarshalObject decodes an object from binary data
func (connection *DbConnection) UnmarshalObject(data []byte, object interface{}) error {
var err error
if connection.getEncryptionKey() != nil {
data, err = decrypt(data, connection.getEncryptionKey())
if err != nil {
return errors.Wrap(err, "Failed decrypting object")
}
}
e := json.Unmarshal(data, object)
if e != nil {
// Special case for the VERSION bucket. Here we're not using json
// So we need to return it as a string
s, ok := object.(*string)
if !ok {
return errors.Wrap(err, e.Error())
}
*s = string(data)
}
return err
}
// UnmarshalObjectWithJsoniter decodes an object from binary data
// using the jsoniter library. It is mainly used to accelerate environment(endpoint)
// decoding at the moment.
func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
if connection.getEncryptionKey() != nil {
var err error
data, err = decrypt(data, connection.getEncryptionKey())
if err != nil {
return err
}
}
var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
err := jsoni.Unmarshal(data, &object)
if err != nil {
if s, ok := object.(*string); ok {
*s = string(data)
return nil
}
return err
}
return nil
}
// mmm, don't have a KMS .... aes GCM seems the most likely from
// https://gist.github.com/atoponce/07d8d4c833873be2f68c34f9afc5a78a#symmetric-encryption
func encrypt(plaintext []byte, passphrase []byte) (encrypted []byte, err error) {
block, _ := aes.NewCipher(passphrase)
gcm, err := cipher.NewGCM(block)
if err != nil {
return encrypted, err
}
nonce := make([]byte, gcm.NonceSize())
if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
return encrypted, err
}
ciphertextByte := gcm.Seal(
nonce,
nonce,
plaintext,
nil)
return ciphertextByte, nil
}
func decrypt(encrypted []byte, passphrase []byte) (plaintextByte []byte, err error) {
if string(encrypted) == "false" {
return []byte("false"), nil
}
block, err := aes.NewCipher(passphrase)
if err != nil {
return encrypted, errors.Wrap(err, "Error creating cypher block")
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return encrypted, errors.Wrap(err, "Error creating GCM")
}
nonceSize := gcm.NonceSize()
if len(encrypted) < nonceSize {
return encrypted, errEncryptedStringTooShort
}
nonce, ciphertextByteClean := encrypted[:nonceSize], encrypted[nonceSize:]
plaintextByte, err = gcm.Open(
nil,
nonce,
ciphertextByteClean,
nil)
if err != nil {
return encrypted, errors.Wrap(err, "Error decrypting text")
}
return plaintextByte, err
}

177
api/database/boltdb/json_test.go
View File

@@ -1,177 +0,0 @@
package boltdb
import (
"crypto/sha256"
"fmt"
"testing"
"github.com/gofrs/uuid"
"github.com/stretchr/testify/assert"
)
const (
jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"InternalAuthSettings": {"RequiredPasswordLength": 12}"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","MPSUser":"","MPSPassword":"","MPSToken":"","CertFileContent":"","CertFileName":"","CertFilePassword":"","DomainName":""},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","EnableTelemetry":true,"HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
passphrase = "my secret key"
)
func secretToEncryptionKey(passphrase string) []byte {
hash := sha256.Sum256([]byte(passphrase))
return hash[:]
}
func Test_MarshalObjectUnencrypted(t *testing.T) {
is := assert.New(t)
uuid := uuid.Must(uuid.NewV4())
tests := []struct {
object interface{}
expected string
}{
{
object: nil,
expected: `null`,
},
{
object: true,
expected: `true`,
},
{
object: false,
expected: `false`,
},
{
object: 123,
expected: `123`,
},
{
object: "456",
expected: "456",
},
{
object: uuid,
expected: "\"" + uuid.String() + "\"",
},
{
object: uuid.String(),
expected: uuid.String(),
},
{
object: map[string]interface{}{"key": "value"},
expected: `{"key":"value"}`,
},
{
object: []bool{true, false},
expected: `[true,false]`,
},
{
object: []int{1, 2, 3},
expected: `[1,2,3]`,
},
{
object: []string{"1", "2", "3"},
expected: `["1","2","3"]`,
},
{
object: []map[string]interface{}{{"key1": "value1"}, {"key2": "value2"}},
expected: `[{"key1":"value1"},{"key2":"value2"}]`,
},
{
object: []interface{}{1, "2", false, map[string]interface{}{"key1": "value1"}},
expected: `[1,"2",false,{"key1":"value1"}]`,
},
}
conn := DbConnection{}
for _, test := range tests {
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
data, err := conn.MarshalObject(test.object)
is.NoError(err)
is.Equal(test.expected, string(data))
})
}
}
func Test_UnMarshalObjectUnencrypted(t *testing.T) {
is := assert.New(t)
// Based on actual data entering and what we expect out of the function
tests := []struct {
object []byte
expected string
}{
{
object: []byte(""),
expected: "",
},
{
object: []byte("35"),
expected: "35",
},
{
// An unmarshalled byte string should return the same without error
object: []byte("9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6"),
expected: "9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6",
},
{
// An un-marshalled json object string should return the same as a string without error also
object: []byte(jsonobject),
expected: jsonobject,
},
}
conn := DbConnection{}
for _, test := range tests {
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
var object string
err := conn.UnmarshalObject(test.object, &object)
is.NoError(err)
is.Equal(test.expected, string(object))
})
}
}
func Test_ObjectMarshallingEncrypted(t *testing.T) {
is := assert.New(t)
// Based on actual data entering and what we expect out of the function
tests := []struct {
object []byte
expected string
}{
{
object: []byte(""),
},
{
object: []byte("35"),
},
{
// An unmarshalled byte string should return the same without error
object: []byte("9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6"),
},
{
// An un-marshalled json object string should return the same as a string without error also
object: []byte(jsonobject),
},
}
key := secretToEncryptionKey(passphrase)
conn := DbConnection{EncryptionKey: key}
for _, test := range tests {
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
data, err := conn.MarshalObject(test.object)
is.NoError(err)
var object []byte
err = conn.UnmarshalObject(data, &object)
is.NoError(err)
is.Equal(test.object, object)
})
}
}

6
api/database/database.go
View File

@@ -4,14 +4,14 @@ import (
"fmt"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/database/boltdb"
"github.com/portainer/portainer/api/database/sqlite"
)
// NewDatabase should use config options to return a connection to the requested database
func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection portainer.Connection, err error) {
switch storeType {
case "boltdb":
return &boltdb.DbConnection{
case "sqlite":
return &sqlite.DbConnection{
Path: storePath,
EncryptionKey: encryptionKey,
}, nil

19
api/database/models/version.go Normal file
View File

@@ -0,0 +1,19 @@
package models
import "net/http"
const (
VersionKey string = "DB_VERSION"
InstanceKey string = "INSTANCE_ID"
EditionKey string = "EDITION"
UpdatingKey string = "DB_UPDATING"
)
type Version struct {
Key string `json:"Key"`
Value string `json:"Value"`
}
func (r *Version) Validate(request *http.Request) error {
return nil
}

149
api/database/sqlite/db.go Normal file
View File

@@ -0,0 +1,149 @@
package sqlite
import (
"errors"
"os"
"path"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"github.com/portainer/portainer/api/database/models"
"github.com/rs/zerolog/log"
)
const (
DatabaseFileName = "portainer.db"
EncryptedDatabaseFileName = "portainer.edb"
)
var (
ErrHaveEncryptedAndUnencrypted = errors.New("Portainer has detected both an encrypted and un-encrypted database and cannot start. Only one database should exist")
ErrHaveEncryptedWithNoKey = errors.New("The portainer database is encrypted, but no secret was loaded")
)
type DbConnection struct {
Path string
EncryptionKey []byte
isEncrypted bool
*gorm.DB
}
func (connection *DbConnection) GetDB() *gorm.DB {
return connection.DB
}
// GetDatabaseFileName get the database filename
func (connection *DbConnection) GetDatabaseFileName() string {
if connection.IsEncryptedStore() {
return EncryptedDatabaseFileName
}
return DatabaseFileName
}
// GetDataseFilePath get the path + filename for the database file
func (connection *DbConnection) GetDatabaseFilePath() string {
if connection.IsEncryptedStore() {
return path.Join(connection.Path, EncryptedDatabaseFileName)
}
return path.Join(connection.Path, DatabaseFileName)
}
// GetStorePath get the filename and path for the database file
func (connection *DbConnection) GetStorePath() string {
return connection.Path
}
func (connection *DbConnection) SetEncrypted(flag bool) {
connection.isEncrypted = flag
}
// Return true if the database is encrypted
func (connection *DbConnection) IsEncryptedStore() bool {
return connection.getEncryptionKey() != nil
}
// NeedsEncryptionMigration returns true if database encryption is enabled and
// we have an un-encrypted DB that requires migration to an encrypted DB
func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
// Cases: Note, we need to check both portainer.db and portainer.edb
// to determine if it's a new store. We only need to differentiate between cases 2,3 and 5
// 1) portainer.edb + key => False
// 2) portainer.edb + no key => ERROR Fatal!
// 3) portainer.db + key => True (needs migration)
// 4) portainer.db + no key => False
// 5) NoDB (new) + key => False
// 6) NoDB (new) + no key => False
// 7) portainer.db & portainer.edb => ERROR Fatal!
// If we have a loaded encryption key, always set encrypted
if connection.EncryptionKey != nil {
connection.SetEncrypted(true)
}
// Check for portainer.db
dbFile := path.Join(connection.Path, DatabaseFileName)
_, err := os.Stat(dbFile)
haveDbFile := err == nil
// Check for portainer.edb
edbFile := path.Join(connection.Path, EncryptedDatabaseFileName)
_, err = os.Stat(edbFile)
haveEdbFile := err == nil
if haveDbFile && haveEdbFile {
// 7 - encrypted and unencrypted db?
return false, ErrHaveEncryptedAndUnencrypted
}
if haveDbFile && connection.EncryptionKey != nil {
// 3 - needs migration
return true, nil
}
if haveEdbFile && connection.EncryptionKey == nil {
// 2 - encrypted db, but no key?
return false, ErrHaveEncryptedWithNoKey
}
// 1, 4, 5, 6
return false, nil
}
// Open opens and initializes the BoltDB database.
func (connection *DbConnection) Open() error {
log.Info().Str("filename", connection.GetDatabaseFileName()).Msg("loading PortainerDB")
// Now we open the db
databasePath := connection.GetDatabaseFilePath()
db, err := gorm.Open(sqlite.Open(databasePath), &gorm.Config{})
if err != nil {
return err
}
connection.DB = db
return nil
}
func (connection *DbConnection) Close() error {
return nil // TODO: check if we need to close the db
}
func (connection *DbConnection) getEncryptionKey() []byte {
if !connection.isEncrypted {
return nil
}
return connection.EncryptionKey
}
func (connection *DbConnection) Init() error {
connection.DB.AutoMigrate(&models.Version{})
return nil
}