WIP: initial POC using a container to call rpc-go to show on the environment page

Signed-off-by: Sven Dowideit <sven.dowideit@portainer.io>

.env.defaults

@@ -1 +0,0 @@
-PORTAINER_EDITION=CE

.eslintrc.yml

@@ -9,7 +9,6 @@ globals:
 
 extends:
   - 'eslint:recommended'
-  - 'plugin:storybook/recommended'
   - prettier
 
 plugins:
@@ -61,7 +60,6 @@ overrides:
       - 'plugin:@typescript-eslint/recommended'
       - 'plugin:@typescript-eslint/eslint-recommended'
       - 'plugin:promise/recommended'
-      - 'plugin:storybook/recommended'
       - prettier # should be last
     settings:
       react:
@@ -79,17 +77,11 @@ overrides:
       react/forbid-prop-types: off
       react/require-default-props: off
       react/no-array-index-key: off
-      no-underscore-dangle: off
       react/jsx-filename-extension: [0]
       import/no-extraneous-dependencies: ['error', { devDependencies: true }]
       '@typescript-eslint/explicit-module-boundary-types': off
       '@typescript-eslint/no-unused-vars': 'error'
       '@typescript-eslint/no-explicit-any': 'error'
-      'jsx-a11y/label-has-associated-control': ['error', { 'assert': 'either' }]
-      'react/function-component-definition': ['error', { 'namedComponents': 'function-declaration' }]
-      'react/jsx-no-bind': off
-      'no-await-in-loop': 'off'
-      'react/jsx-no-useless-fragment': ['error', { allowExpressions: true }]
   - files:
       - app/**/*.test.*
     extends:
@@ -97,9 +89,3 @@ overrides:
       - 'plugin:jest/style'
     env:
       'jest/globals': true
-    rules:
-      'react/jsx-no-constructed-context-values': off
-  - files:
-      - app/**/*.stories.*
-    rules:
-      'no-alert': off

.git-blame-ignore-revs

@@ -1,5 +0,0 @@
-# prettier
-cf5056d9c03b62d91a25c3b9127caac838695f98
-
-# prettier v2
-42e7db0ae7897d3cb72b0ea1ecf57ee2dd694169

.github/ISSUE_TEMPLATE.md

@@ -2,7 +2,7 @@
 
 Thanks for opening an issue on Portainer !
 
-Do you need help or have a question? Come chat with us on Slack https://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
+Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
 
 If you are reporting a new issue, make sure that we do not have any duplicates
 already open. You can ensure this by searching the issue list for this

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -12,7 +12,7 @@ Thanks for reporting a bug for Portainer !
 
 You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
 
-Do you need help or have a question? Come chat with us on Slack https://portainer.io/slack/
+Do you need help or have a question? Come chat with us on Slack http://portainer.slack.com/
 
 Before opening a new issue, make sure that we do not have any duplicates
 already open. You can ensure this by searching the issue list for this
@@ -47,7 +47,7 @@ You can see how [here](https://documentation.portainer.io/r/portainer-logs)
 - Platform (windows/linux):
 - Command used to start Portainer (`docker run -p 9443:9443 portainer/portainer`):
 - Browser:
-- Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commercial setup.
+- Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commerical setup.
 - Have you reviewed our technical documentation and knowledge base? Yes/No
 
 **Additional context**

.github/ISSUE_TEMPLATE/Custom.md

@@ -4,11 +4,11 @@ about: Ask us a question about Portainer usage or deployment
 title: ''
 labels: ''
 assignees: ''
----
 
+---
 Before you start, we need a little bit more information from you:
 
-Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commercial setup.
+Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commerical setup.
 
 Have you reviewed our technical documentation and knowledge base? Yes/No
 
@@ -16,7 +16,7 @@ Have you reviewed our technical documentation and knowledge base? Yes/No
 
 You can find more information about Portainer support framework policy here: https://old.portainer.io/2019/04/portainer-support-policy/
 
-Do you need help or have a question? Come chat with us on Slack https://portainer.io/slack/
+Do you need help or have a question? Come chat with us on Slack http://portainer.slack.com/
 
 Also, be sure to check our FAQ and documentation first: https://documentation.portainer.io/
 -->

.github/ISSUE_TEMPLATE/Feature_request.md

@@ -4,13 +4,14 @@ about: Suggest a feature/enhancement that should be added in Portainer
 title: ''
 labels: ''
 assignees: ''
+
 ---
 
 <!--
 
 Thanks for opening a feature request for Portainer !
 
-Do you need help or have a question? Come chat with us on Slack https://portainer.io/slack/
+Do you need help or have a question? Come chat with us on Slack http://portainer.slack.com/
 
 Before opening a new issue, make sure that we do not have any duplicates
 already open. You can ensure this by searching the issue list for this

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +0,0 @@
-closes #0 <!-- Github issue number (remove if unknown) -->
-closes [CE-0] <!-- Jira link number (remove if unknown). Please also add the same [CE-XXX] at the back of the PR title -->
-
-### Changes:

.github/workflows/lint.yml

@@ -1,41 +0,0 @@
-name: Lint
-
-on:
-  push:
-    branches:
-      - master
-      - develop
-      - release/*
-  pull_request:
-    branches:
-      - master
-      - develop
-      - release/*
-
-jobs:
-  run-linters:
-    name: Run linters
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Check out Git repository
-        uses: actions/checkout@v2
-
-      - name: Set up Node.js
-        uses: actions/setup-node@v1
-        with:
-          node-version: 12
-
-      # ESLint and Prettier must be in `package.json`
-      - name: Install Node.js dependencies
-        run: yarn --frozen-lockfile
-
-      - name: Run linters
-        uses: wearerequired/lint-action@v1
-        with:
-          eslint: true
-          eslint_extensions: ts,tsx,js,jsx
-          prettier: true
-          prettier_dir: app/
-          gofmt: true
-          gofmt_dir: api/

.github/workflows/test-client.yaml

@@ -1,11 +0,0 @@
-name: Test Frontend
-on: push
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install modules
-        run: yarn --frozen-lockfile
-      - name: Run tests
-        run: yarn test:client

.github/workflows/validate-openapi-spec.yml

@@ -1,53 +0,0 @@
-name: Validate
-
-on:
-  pull_request:
-    branches:
-      - master
-      - develop
-      - 'release/*'
-
-jobs:
-  openapi-spec:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout Code
-      uses: actions/checkout@v2
-    - name: Setup Node v14
-      uses: actions/setup-node@v2
-      with:
-        node-version: 14
-
-    # https://github.com/actions/cache/blob/main/examples.md#node---yarn
-    - name: Get yarn cache directory path
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-
-    - uses: actions/cache@v2
-      id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-
-    - name: Setup Go v1.17.3
-      uses: actions/setup-go@v2
-      with:
-        go-version: '^1.17.3'
-
-    - name: Prebuild docs
-      run: yarn prebuild:docs
-
-    - name: Build OpenAPI 2.0 Spec
-      run: yarn build:docs
-
-    # Install dependencies globally to bypass installing all frontend deps
-    - name: Install swagger2openapi and swagger-cli
-      run: yarn global add swagger2openapi @apidevtools/swagger-cli
-
-    # OpenAPI2.0 does not support multiple body params (which we utilise in some of our handlers).
-    # OAS3.0 however does support multiple body params - hence its best to convert the generated OAS 2.0
-    # to OAS 3.0 and validate the output of generated OAS 3.0 instead.
-    - name: Convert OpenAPI 2.0 to OpenAPI 3.0 and validate spec
-      run: yarn validate:docs

.gitignore

@@ -7,7 +7,6 @@ storybook-static
 .tmp
 **/.vscode/settings.json
 **/.vscode/tasks.json
-*.DS_Store
 
 .eslintcache
 __debug_bin

.prettierrc

@@ -4,16 +4,20 @@
   "htmlWhitespaceSensitivity": "strict",
   "overrides": [
     {
-      "files": ["*.html"],
+      "files": [
+        "*.html"
+      ],
       "options": {
         "parser": "angular"
       }
     },
     {
-      "files": ["*.{j,t}sx", "*.ts"],
+      "files": [
+        "*.{j,t}sx"
+      ],
       "options": {
-        "printWidth": 80
+        "printWidth": 80,
       }
     }
   ]
-}
+}

.storybook/main.js

@@ -28,8 +28,4 @@ module.exports = {
     ];
     return config;
   },
-  core: {
-    builder: 'webpack5',
-  },
-  staticDirs: ['./public'],
 };

.storybook/preview.js

@@ -1,23 +1,5 @@
 import '../app/assets/css';
 
-import { pushStateLocationPlugin, UIRouter } from '@uirouter/react';
-import { initialize as initMSW, mswDecorator } from 'msw-storybook-addon';
-import { handlers } from '@/setup-tests/server-handlers';
-
-// Initialize MSW
-initMSW({
-  onUnhandledRequest: ({ method, url }) => {
-    if (url.pathname.startsWith('/api')) {
-      console.error(`Unhandled ${method} request to ${url}.
-
-        This exception has been only logged in the console, however, it's strongly recommended to resolve this error as you don't want unmocked data in Storybook stories.
-
-        If you wish to mock an error response, please refer to this guide: https://mswjs.io/docs/recipes/mocking-error-responses
-      `);
-    }
-  },
-});
-
 export const parameters = {
   actions: { argTypesRegex: '^on[A-Z].*' },
   controls: {
@@ -26,16 +8,4 @@ export const parameters = {
       date: /Date$/,
     },
   },
-  msw: {
-    handlers,
-  },
 };
-
-export const decorators = [
-  (Story) => (
-    <UIRouter plugins={[pushStateLocationPlugin]}>
-      <Story />
-    </UIRouter>
-  ),
-  mswDecorator,
-];

.storybook/public/mockServiceWorker.js

@@ -1,328 +0,0 @@
-/* eslint-disable */
-/* tslint:disable */
-
-/**
- * Mock Service Worker (0.36.3).
- * @see https://github.com/mswjs/msw
- * - Please do NOT modify this file.
- * - Please do NOT serve this file on production.
- */
-
-const INTEGRITY_CHECKSUM = '02f4ad4a2797f85668baf196e553d929';
-const bypassHeaderName = 'x-msw-bypass';
-const activeClientIds = new Set();
-
-self.addEventListener('install', function () {
-  return self.skipWaiting();
-});
-
-self.addEventListener('activate', async function (event) {
-  return self.clients.claim();
-});
-
-self.addEventListener('message', async function (event) {
-  const clientId = event.source.id;
-
-  if (!clientId || !self.clients) {
-    return;
-  }
-
-  const client = await self.clients.get(clientId);
-
-  if (!client) {
-    return;
-  }
-
-  const allClients = await self.clients.matchAll();
-
-  switch (event.data) {
-    case 'KEEPALIVE_REQUEST': {
-      sendToClient(client, {
-        type: 'KEEPALIVE_RESPONSE',
-      });
-      break;
-    }
-
-    case 'INTEGRITY_CHECK_REQUEST': {
-      sendToClient(client, {
-        type: 'INTEGRITY_CHECK_RESPONSE',
-        payload: INTEGRITY_CHECKSUM,
-      });
-      break;
-    }
-
-    case 'MOCK_ACTIVATE': {
-      activeClientIds.add(clientId);
-
-      sendToClient(client, {
-        type: 'MOCKING_ENABLED',
-        payload: true,
-      });
-      break;
-    }
-
-    case 'MOCK_DEACTIVATE': {
-      activeClientIds.delete(clientId);
-      break;
-    }
-
-    case 'CLIENT_CLOSED': {
-      activeClientIds.delete(clientId);
-
-      const remainingClients = allClients.filter((client) => {
-        return client.id !== clientId;
-      });
-
-      // Unregister itself when there are no more clients
-      if (remainingClients.length === 0) {
-        self.registration.unregister();
-      }
-
-      break;
-    }
-  }
-});
-
-// Resolve the "main" client for the given event.
-// Client that issues a request doesn't necessarily equal the client
-// that registered the worker. It's with the latter the worker should
-// communicate with during the response resolving phase.
-async function resolveMainClient(event) {
-  const client = await self.clients.get(event.clientId);
-
-  if (client.frameType === 'top-level') {
-    return client;
-  }
-
-  const allClients = await self.clients.matchAll();
-
-  return allClients
-    .filter((client) => {
-      // Get only those clients that are currently visible.
-      return client.visibilityState === 'visible';
-    })
-    .find((client) => {
-      // Find the client ID that's recorded in the
-      // set of clients that have registered the worker.
-      return activeClientIds.has(client.id);
-    });
-}
-
-async function handleRequest(event, requestId) {
-  const client = await resolveMainClient(event);
-  const response = await getResponse(event, client, requestId);
-
-  // Send back the response clone for the "response:*" life-cycle events.
-  // Ensure MSW is active and ready to handle the message, otherwise
-  // this message will pend indefinitely.
-  if (client && activeClientIds.has(client.id)) {
-    (async function () {
-      const clonedResponse = response.clone();
-      sendToClient(client, {
-        type: 'RESPONSE',
-        payload: {
-          requestId,
-          type: clonedResponse.type,
-          ok: clonedResponse.ok,
-          status: clonedResponse.status,
-          statusText: clonedResponse.statusText,
-          body: clonedResponse.body === null ? null : await clonedResponse.text(),
-          headers: serializeHeaders(clonedResponse.headers),
-          redirected: clonedResponse.redirected,
-        },
-      });
-    })();
-  }
-
-  return response;
-}
-
-async function getResponse(event, client, requestId) {
-  const { request } = event;
-  const requestClone = request.clone();
-  const getOriginalResponse = () => fetch(requestClone);
-
-  // Bypass mocking when the request client is not active.
-  if (!client) {
-    return getOriginalResponse();
-  }
-
-  // Bypass initial page load requests (i.e. static assets).
-  // The absence of the immediate/parent client in the map of the active clients
-  // means that MSW hasn't dispatched the "MOCK_ACTIVATE" event yet
-  // and is not ready to handle requests.
-  if (!activeClientIds.has(client.id)) {
-    return await getOriginalResponse();
-  }
-
-  // Bypass requests with the explicit bypass header
-  if (requestClone.headers.get(bypassHeaderName) === 'true') {
-    const cleanRequestHeaders = serializeHeaders(requestClone.headers);
-
-    // Remove the bypass header to comply with the CORS preflight check.
-    delete cleanRequestHeaders[bypassHeaderName];
-
-    const originalRequest = new Request(requestClone, {
-      headers: new Headers(cleanRequestHeaders),
-    });
-
-    return fetch(originalRequest);
-  }
-
-  // Send the request to the client-side MSW.
-  const reqHeaders = serializeHeaders(request.headers);
-  const body = await request.text();
-
-  const clientMessage = await sendToClient(client, {
-    type: 'REQUEST',
-    payload: {
-      id: requestId,
-      url: request.url,
-      method: request.method,
-      headers: reqHeaders,
-      cache: request.cache,
-      mode: request.mode,
-      credentials: request.credentials,
-      destination: request.destination,
-      integrity: request.integrity,
-      redirect: request.redirect,
-      referrer: request.referrer,
-      referrerPolicy: request.referrerPolicy,
-      body,
-      bodyUsed: request.bodyUsed,
-      keepalive: request.keepalive,
-    },
-  });
-
-  switch (clientMessage.type) {
-    case 'MOCK_SUCCESS': {
-      return delayPromise(() => respondWithMock(clientMessage), clientMessage.payload.delay);
-    }
-
-    case 'MOCK_NOT_FOUND': {
-      return getOriginalResponse();
-    }
-
-    case 'NETWORK_ERROR': {
-      const { name, message } = clientMessage.payload;
-      const networkError = new Error(message);
-      networkError.name = name;
-
-      // Rejecting a request Promise emulates a network error.
-      throw networkError;
-    }
-
-    case 'INTERNAL_ERROR': {
-      const parsedBody = JSON.parse(clientMessage.payload.body);
-
-      console.error(
-        `\
-[MSW] Uncaught exception in the request handler for "%s %s":
-
-${parsedBody.location}
-
-This exception has been gracefully handled as a 500 response, however, it's strongly recommended to resolve this error, as it indicates a mistake in your code. If you wish to mock an error response, please see this guide: https://mswjs.io/docs/recipes/mocking-error-responses\
-`,
-        request.method,
-        request.url
-      );
-
-      return respondWithMock(clientMessage);
-    }
-  }
-
-  return getOriginalResponse();
-}
-
-self.addEventListener('fetch', function (event) {
-  const { request } = event;
-  const accept = request.headers.get('accept') || '';
-
-  // Bypass server-sent events.
-  if (accept.includes('text/event-stream')) {
-    return;
-  }
-
-  // Bypass navigation requests.
-  if (request.mode === 'navigate') {
-    return;
-  }
-
-  // Opening the DevTools triggers the "only-if-cached" request
-  // that cannot be handled by the worker. Bypass such requests.
-  if (request.cache === 'only-if-cached' && request.mode !== 'same-origin') {
-    return;
-  }
-
-  // Bypass all requests when there are no active clients.
-  // Prevents the self-unregistered worked from handling requests
-  // after it's been deleted (still remains active until the next reload).
-  if (activeClientIds.size === 0) {
-    return;
-  }
-
-  const requestId = uuidv4();
-
-  return event.respondWith(
-    handleRequest(event, requestId).catch((error) => {
-      if (error.name === 'NetworkError') {
-        console.warn('[MSW] Successfully emulated a network error for the "%s %s" request.', request.method, request.url);
-        return;
-      }
-
-      // At this point, any exception indicates an issue with the original request/response.
-      console.error(
-        `\
-[MSW] Caught an exception from the "%s %s" request (%s). This is probably not a problem with Mock Service Worker. There is likely an additional logging output above.`,
-        request.method,
-        request.url,
-        `${error.name}: ${error.message}`
-      );
-    })
-  );
-});
-
-function serializeHeaders(headers) {
-  const reqHeaders = {};
-  headers.forEach((value, name) => {
-    reqHeaders[name] = reqHeaders[name] ? [].concat(reqHeaders[name]).concat(value) : value;
-  });
-  return reqHeaders;
-}
-
-function sendToClient(client, message) {
-  return new Promise((resolve, reject) => {
-    const channel = new MessageChannel();
-
-    channel.port1.onmessage = (event) => {
-      if (event.data && event.data.error) {
-        return reject(event.data.error);
-      }
-
-      resolve(event.data);
-    };
-
-    client.postMessage(JSON.stringify(message), [channel.port2]);
-  });
-}
-
-function delayPromise(cb, duration) {
-  return new Promise((resolve) => {
-    setTimeout(() => resolve(cb()), duration);
-  });
-}
-
-function respondWithMock(clientMessage) {
-  return new Response(clientMessage.payload.body, {
-    ...clientMessage.payload,
-    headers: clientMessage.payload.headers,
-  });
-}
-
-function uuidv4() {
-  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
-    const r = (Math.random() * 16) | 0;
-    const v = c == 'x' ? r : (r & 0x3) | 0x8;
-    return v.toString(16);
-  });
-}

.vscode.example/launch.json

@@ -9,7 +9,7 @@
       "type": "go",
       "request": "launch",
       "mode": "debug",
-      "program": "${workspaceRoot}/api/cmd/portainer",
+      "program": "${workspaceRoot}/api/cmd/portainer/main.go",
       "cwd": "${workspaceRoot}",
       "env": {},
       "showLog": true,

.vscode.example/portainer.code-snippets

@@ -150,7 +150,6 @@
       "// @description ",
       "// @description **Access policy**: ",
       "// @tags ",
-      "// @security ApiKeyAuth",
       "// @security jwt",
       "// @accept json",
       "// @produce json",

.vscode.example/settings.json

@@ -1,8 +1,4 @@
 {
   "go.lintTool": "golangci-lint",
-  "go.lintFlags": ["--fast", "-E", "exportloopref"],
-  "gopls": {
-    "build.expandWorkspaceToModule": false
-  },
-  "gitlens.advanced.blame.customArguments": ["--ignore-revs-file", ".git-blame-ignore-revs"]
+  "go.lintFlags": ["--fast", "-E", "exportloopref"]
 }

CONTRIBUTING.md

@@ -97,11 +97,13 @@ Find more detailed steps at <https://documentation.portainer.io/contributing/ins
 
 ### Build customisation
 
+By default, `yarn start` will use `/tmp/portainer/` for the data store, so it won't persist over reboots.
+
 You can customise the following settings:
 
-- `PORTAINER_DATA`: The host dir or volume name used by portainer (default is `/tmp/portainer`, which won't persist over reboots).
-- `PORTAINER_PROJECT`: The root dir of the repository - `${portainerRoot}/dist/` is imported into the container to get the build artifacts and external tools (defaults to `your current dir`).
-- `PORTAINER_FLAGS`: a list of flags to be used on the portainer commandline, in the form `--admin-password=<pwd hash> --feat fdo=false --feat open-amt` (default: `""`).
+- `PORTAINER_DATA`: The host dir or volume name used by portainer (default `/tmp/portainer`)
+- `PORTAINER_PROJECT`: The root dir of the repository - `${portainerRoot}/dist/` is imported into the container to get the build artifacts and external tools (defaults to `your current dir`)
+- `PORTAINER_FLAGS`: a list of flags to be used on the portainer commandline, in the form `--admin-password=<pwd hash> --feat fdo=false --feat open-amt` (default: `""`)
 
 ## Adding api docs
 
@@ -120,7 +122,6 @@ When adding a new route to an existing handler use the following as a template (
 // @description
 // @description **Access policy**:
 // @tags
-// @security ApiKeyAuth
 // @security jwt
 // @accept json
 // @produce json

README.md

@@ -42,10 +42,10 @@ View [this](https://www.portainer.io/products) table to see all of the Portainer
 
 Portainer CE is an open source project and is supported by the community. You can buy a supported version of Portainer at portainer.io
 
-Learn more about Portainer's community support channels [here.](https://www.portainer.io/community_help)
+Learn more about Portainers community support channels [here.](https://www.portainer.io/community_help)
 
 - Issues: https://github.com/portainer/portainer/issues
-- Slack (chat): [https://portainer.io/slack](https://portainer.io/slack)
+- Slack (chat): [https://portainer.slack.com/](https://join.slack.com/t/portainer/shared_invite/zt-txh3ljab-52QHTyjCqbe5RibC2lcjKA)
 
 You can join the Portainer Community by visiting community.portainer.io. This will give you advance notice of events, content and other related Portainer content.
 

api/adminmonitor/admin_monitor.go

@@ -6,20 +6,19 @@ import (
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
 )
 
 var logFatalf = log.Fatalf
 
 type Monitor struct {
 	timeout          time.Duration
-	datastore        dataservices.DataStore
+	datastore        portainer.DataStore
 	shutdownCtx      context.Context
 	cancellationFunc context.CancelFunc
 }
 
 // New creates a monitor that when started will wait for the timeout duration and then shutdown the application unless it has been initialized.
-func New(timeout time.Duration, datastore dataservices.DataStore, shutdownCtx context.Context) *Monitor {
+func New(timeout time.Duration, datastore portainer.DataStore, shutdownCtx context.Context) *Monitor {
 	return &Monitor{
 		timeout:     timeout,
 		datastore:   datastore,

api/apikey/apikey.go

@@ -1,30 +0,0 @@
-package apikey
-
-import (
-	"crypto/rand"
-	"io"
-
-	portainer "github.com/portainer/portainer/api"
-)
-
-// APIKeyService represents a service for managing API keys.
-type APIKeyService interface {
-	HashRaw(rawKey string) []byte
-	GenerateApiKey(user portainer.User, description string) (string, *portainer.APIKey, error)
-	GetAPIKey(apiKeyID portainer.APIKeyID) (*portainer.APIKey, error)
-	GetAPIKeys(userID portainer.UserID) ([]portainer.APIKey, error)
-	GetDigestUserAndKey(digest []byte) (portainer.User, portainer.APIKey, error)
-	UpdateAPIKey(apiKey *portainer.APIKey) error
-	DeleteAPIKey(apiKeyID portainer.APIKeyID) error
-	InvalidateUserKeyCache(userId portainer.UserID) bool
-}
-
-// generateRandomKey generates a random key of specified length
-// source: https://github.com/gorilla/securecookie/blob/master/securecookie.go#L515
-func generateRandomKey(length int) []byte {
-	k := make([]byte, length)
-	if _, err := io.ReadFull(rand.Reader, k); err != nil {
-		return nil
-	}
-	return k
-}

api/apikey/apikey_test.go

@@ -1,50 +0,0 @@
-package apikey
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_generateRandomKey(t *testing.T) {
-	is := assert.New(t)
-
-	tests := []struct {
-		name      string
-		wantLenth int
-	}{
-		{
-			name:      "Generate a random key of length 16",
-			wantLenth: 16,
-		},
-		{
-			name:      "Generate a random key of length 32",
-			wantLenth: 32,
-		},
-		{
-			name:      "Generate a random key of length 64",
-			wantLenth: 64,
-		},
-		{
-			name:      "Generate a random key of length 128",
-			wantLenth: 128,
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := generateRandomKey(tt.wantLenth)
-			is.Equal(tt.wantLenth, len(got))
-		})
-	}
-
-	t.Run("Generated keys are unique", func(t *testing.T) {
-		keys := make(map[string]bool)
-		for i := 0; i < 100; i++ {
-			key := generateRandomKey(8)
-			_, ok := keys[string(key)]
-			is.False(ok)
-			keys[string(key)] = true
-		}
-	})
-}

api/apikey/cache.go

@@ -1,69 +0,0 @@
-package apikey
-
-import (
-	lru "github.com/hashicorp/golang-lru"
-	portainer "github.com/portainer/portainer/api"
-)
-
-const defaultAPIKeyCacheSize = 1024
-
-// entry is a tuple containing the user and API key associated to an API key digest
-type entry struct {
-	user   portainer.User
-	apiKey portainer.APIKey
-}
-
-// apiKeyCache is a concurrency-safe, in-memory cache which primarily exists for to reduce database roundtrips.
-// We store the api-key digest (keys) and the associated user and key-data (values) in the cache.
-// This is required because HTTP requests will contain only the api-key digest in the x-api-key request header;
-// digest value must be mapped to a portainer user (and respective key data) for validation.
-// This cache is used to avoid multiple database queries to retrieve these user/key associated to the digest.
-type apiKeyCache struct {
-	// cache type [string]entry cache (key: string(digest), value: user/key entry)
-	// note: []byte keys are not supported by golang-lru Cache
-	cache *lru.Cache
-}
-
-// NewAPIKeyCache creates a new cache for API keys
-func NewAPIKeyCache(cacheSize int) *apiKeyCache {
-	cache, _ := lru.New(cacheSize)
-	return &apiKeyCache{cache: cache}
-}
-
-// Get returns the user/key associated to an api-key's digest
-// This is required because HTTP requests will contain the digest of the API key in header,
-// the digest value must be mapped to a portainer user.
-func (c *apiKeyCache) Get(digest []byte) (portainer.User, portainer.APIKey, bool) {
-	val, ok := c.cache.Get(string(digest))
-	if !ok {
-		return portainer.User{}, portainer.APIKey{}, false
-	}
-	tuple := val.(entry)
-
-	return tuple.user, tuple.apiKey, true
-}
-
-// Set persists a user/key entry to the cache
-func (c *apiKeyCache) Set(digest []byte, user portainer.User, apiKey portainer.APIKey) {
-	c.cache.Add(string(digest), entry{
-		user:   user,
-		apiKey: apiKey,
-	})
-}
-
-// Delete evicts a digest's user/key entry key from the cache
-func (c *apiKeyCache) Delete(digest []byte) {
-	c.cache.Remove(string(digest))
-}
-
-// InvalidateUserKeyCache loops through all the api-keys associated to a user and removes them from the cache
-func (c *apiKeyCache) InvalidateUserKeyCache(userId portainer.UserID) bool {
-	present := false
-	for _, k := range c.cache.Keys() {
-		user, _, _ := c.Get([]byte(k.(string)))
-		if user.ID == userId {
-			present = c.cache.Remove(k)
-		}
-	}
-	return present
-}

api/apikey/cache_test.go

@@ -1,181 +0,0 @@
-package apikey
-
-import (
-	"testing"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_apiKeyCacheGet(t *testing.T) {
-	is := assert.New(t)
-
-	keyCache := NewAPIKeyCache(10)
-
-	// pre-populate cache
-	keyCache.cache.Add(string("foo"), entry{user: portainer.User{}, apiKey: portainer.APIKey{}})
-	keyCache.cache.Add(string(""), entry{user: portainer.User{}, apiKey: portainer.APIKey{}})
-
-	tests := []struct {
-		digest []byte
-		found  bool
-	}{
-		{
-			digest: []byte("foo"),
-			found:  true,
-		},
-		{
-			digest: []byte(""),
-			found:  true,
-		},
-		{
-			digest: []byte("bar"),
-			found:  false,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(string(test.digest), func(t *testing.T) {
-			_, _, found := keyCache.Get(test.digest)
-			is.Equal(test.found, found)
-		})
-	}
-}
-
-func Test_apiKeyCacheSet(t *testing.T) {
-	is := assert.New(t)
-
-	keyCache := NewAPIKeyCache(10)
-
-	// pre-populate cache
-	keyCache.Set([]byte("bar"), portainer.User{ID: 2}, portainer.APIKey{})
-	keyCache.Set([]byte("foo"), portainer.User{ID: 1}, portainer.APIKey{})
-
-	// overwrite existing entry
-	keyCache.Set([]byte("foo"), portainer.User{ID: 3}, portainer.APIKey{})
-
-	val, ok := keyCache.cache.Get(string("bar"))
-	is.True(ok)
-
-	tuple := val.(entry)
-	is.Equal(portainer.User{ID: 2}, tuple.user)
-
-	val, ok = keyCache.cache.Get(string("foo"))
-	is.True(ok)
-
-	tuple = val.(entry)
-	is.Equal(portainer.User{ID: 3}, tuple.user)
-}
-
-func Test_apiKeyCacheDelete(t *testing.T) {
-	is := assert.New(t)
-
-	keyCache := NewAPIKeyCache(10)
-
-	t.Run("Delete an existing entry", func(t *testing.T) {
-		keyCache.cache.Add(string("foo"), entry{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
-		keyCache.Delete([]byte("foo"))
-
-		_, ok := keyCache.cache.Get(string("foo"))
-		is.False(ok)
-	})
-
-	t.Run("Delete a non-existing entry", func(t *testing.T) {
-		nonPanicFunc := func() { keyCache.Delete([]byte("non-existent-key")) }
-		is.NotPanics(nonPanicFunc)
-	})
-}
-
-func Test_apiKeyCacheLRU(t *testing.T) {
-	is := assert.New(t)
-
-	tests := []struct {
-		name        string
-		cacheLen    int
-		key         []string
-		foundKeys   []string
-		evictedKeys []string
-	}{
-		{
-			name:        "Cache length is 1, add 2 keys",
-			cacheLen:    1,
-			key:         []string{"foo", "bar"},
-			foundKeys:   []string{"bar"},
-			evictedKeys: []string{"foo"},
-		},
-		{
-			name:        "Cache length is 1, add 3 keys",
-			cacheLen:    1,
-			key:         []string{"foo", "bar", "baz"},
-			foundKeys:   []string{"baz"},
-			evictedKeys: []string{"foo", "bar"},
-		},
-		{
-			name:        "Cache length is 2, add 3 keys",
-			cacheLen:    2,
-			key:         []string{"foo", "bar", "baz"},
-			foundKeys:   []string{"bar", "baz"},
-			evictedKeys: []string{"foo"},
-		},
-		{
-			name:        "Cache length is 2, add 4 keys",
-			cacheLen:    2,
-			key:         []string{"foo", "bar", "baz", "qux"},
-			foundKeys:   []string{"baz", "qux"},
-			evictedKeys: []string{"foo", "bar"},
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			keyCache := NewAPIKeyCache(test.cacheLen)
-
-			for _, key := range test.key {
-				keyCache.Set([]byte(key), portainer.User{ID: 1}, portainer.APIKey{})
-			}
-
-			for _, key := range test.foundKeys {
-				_, _, found := keyCache.Get([]byte(key))
-				is.True(found, "Key %s not found", key)
-			}
-
-			for _, key := range test.evictedKeys {
-				_, _, found := keyCache.Get([]byte(key))
-				is.False(found, "key %s should have been evicted", key)
-			}
-		})
-	}
-}
-
-func Test_apiKeyCacheInvalidateUserKeyCache(t *testing.T) {
-	is := assert.New(t)
-
-	keyCache := NewAPIKeyCache(10)
-
-	t.Run("Removes users keys from cache", func(t *testing.T) {
-		keyCache.cache.Add(string("foo"), entry{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
-
-		ok := keyCache.InvalidateUserKeyCache(1)
-		is.True(ok)
-
-		_, ok = keyCache.cache.Get(string("foo"))
-		is.False(ok)
-	})
-
-	t.Run("Does not affect other keys", func(t *testing.T) {
-		keyCache.cache.Add(string("foo"), entry{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
-		keyCache.cache.Add(string("bar"), entry{user: portainer.User{ID: 2}, apiKey: portainer.APIKey{}})
-
-		ok := keyCache.InvalidateUserKeyCache(1)
-		is.True(ok)
-
-		ok = keyCache.InvalidateUserKeyCache(1)
-		is.False(ok)
-
-		_, ok = keyCache.cache.Get(string("foo"))
-		is.False(ok)
-
-		_, ok = keyCache.cache.Get(string("bar"))
-		is.True(ok)
-	})
-}

api/apikey/service.go

@@ -1,127 +0,0 @@
-package apikey
-
-import (
-	"crypto/sha256"
-	"encoding/base64"
-	"fmt"
-	"time"
-
-	"github.com/pkg/errors"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
-)
-
-const portainerAPIKeyPrefix = "ptr_"
-
-var ErrInvalidAPIKey = errors.New("Invalid API key")
-
-type apiKeyService struct {
-	apiKeyRepository dataservices.APIKeyRepository
-	userRepository   dataservices.UserService
-	cache            *apiKeyCache
-}
-
-func NewAPIKeyService(apiKeyRepository dataservices.APIKeyRepository, userRepository dataservices.UserService) *apiKeyService {
-	return &apiKeyService{
-		apiKeyRepository: apiKeyRepository,
-		userRepository:   userRepository,
-		cache:            NewAPIKeyCache(defaultAPIKeyCacheSize),
-	}
-}
-
-// HashRaw computes a hash digest of provided raw API key.
-func (a *apiKeyService) HashRaw(rawKey string) []byte {
-	hashDigest := sha256.Sum256([]byte(rawKey))
-	return hashDigest[:]
-}
-
-// GenerateApiKey generates a raw API key for a user (for one-time display).
-// The generated API key is stored in the cache and database.
-func (a *apiKeyService) GenerateApiKey(user portainer.User, description string) (string, *portainer.APIKey, error) {
-	randKey := generateRandomKey(32)
-	encodedRawAPIKey := base64.StdEncoding.EncodeToString(randKey)
-	prefixedAPIKey := portainerAPIKeyPrefix + encodedRawAPIKey
-
-	hashDigest := a.HashRaw(prefixedAPIKey)
-
-	apiKey := &portainer.APIKey{
-		UserID:      user.ID,
-		Description: description,
-		Prefix:      prefixedAPIKey[:7],
-		DateCreated: time.Now().Unix(),
-		Digest:      hashDigest,
-	}
-
-	err := a.apiKeyRepository.CreateAPIKey(apiKey)
-	if err != nil {
-		return "", nil, errors.Wrap(err, "Unable to create API key")
-	}
-
-	// persist api-key to cache
-	a.cache.Set(apiKey.Digest, user, *apiKey)
-
-	return prefixedAPIKey, apiKey, nil
-}
-
-// GetAPIKey returns an API key by its ID.
-func (a *apiKeyService) GetAPIKey(apiKeyID portainer.APIKeyID) (*portainer.APIKey, error) {
-	return a.apiKeyRepository.GetAPIKey(apiKeyID)
-}
-
-// GetAPIKeys returns all the API keys associated to a user.
-func (a *apiKeyService) GetAPIKeys(userID portainer.UserID) ([]portainer.APIKey, error) {
-	return a.apiKeyRepository.GetAPIKeysByUserID(userID)
-}
-
-// GetDigestUserAndKey returns the user and api-key associated to a specified hash digest.
-// A cache lookup is performed first; if the user/api-key is not found in the cache, respective database lookups are performed.
-func (a *apiKeyService) GetDigestUserAndKey(digest []byte) (portainer.User, portainer.APIKey, error) {
-	// get api key from cache if possible
-	cachedUser, cachedKey, ok := a.cache.Get(digest)
-	if ok {
-		return cachedUser, cachedKey, nil
-	}
-
-	apiKey, err := a.apiKeyRepository.GetAPIKeyByDigest(digest)
-	if err != nil {
-		return portainer.User{}, portainer.APIKey{}, errors.Wrap(err, "Unable to retrieve API key")
-	}
-
-	user, err := a.userRepository.User(apiKey.UserID)
-	if err != nil {
-		return portainer.User{}, portainer.APIKey{}, errors.Wrap(err, "Unable to retrieve digest user")
-	}
-
-	// persist api-key to cache - for quicker future lookups
-	a.cache.Set(apiKey.Digest, *user, *apiKey)
-
-	return *user, *apiKey, nil
-}
-
-// UpdateAPIKey updates an API key and in cache and database.
-func (a *apiKeyService) UpdateAPIKey(apiKey *portainer.APIKey) error {
-	user, _, err := a.GetDigestUserAndKey(apiKey.Digest)
-	if err != nil {
-		return errors.Wrap(err, "Unable to retrieve API key")
-	}
-	a.cache.Set(apiKey.Digest, user, *apiKey)
-	return a.apiKeyRepository.UpdateAPIKey(apiKey)
-}
-
-// DeleteAPIKey deletes an API key and removes the digest/api-key entry from the cache.
-func (a *apiKeyService) DeleteAPIKey(apiKeyID portainer.APIKeyID) error {
-	// get api-key digest to remove from cache
-	apiKey, err := a.apiKeyRepository.GetAPIKey(apiKeyID)
-	if err != nil {
-		return errors.Wrap(err, fmt.Sprintf("Unable to retrieve API key: %d", apiKeyID))
-	}
-
-	// delete the user/api-key from cache
-	a.cache.Delete(apiKey.Digest)
-	return a.apiKeyRepository.DeleteAPIKey(apiKeyID)
-}
-
-func (a *apiKeyService) InvalidateUserKeyCache(userId portainer.UserID) bool {
-	return a.cache.InvalidateUserKeyCache(userId)
-}

api/apikey/service_test.go

@@ -1,309 +0,0 @@
-package apikey
-
-import (
-	"crypto/sha256"
-	"log"
-	"strings"
-	"testing"
-	"time"
-
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/datastore"
-	"github.com/stretchr/testify/assert"
-)
-
-func Test_SatisfiesAPIKeyServiceInterface(t *testing.T) {
-	is := assert.New(t)
-	is.Implements((*APIKeyService)(nil), NewAPIKeyService(nil, nil))
-}
-
-func Test_GenerateApiKey(t *testing.T) {
-	is := assert.New(t)
-
-	_, store, teardown := datastore.MustNewTestStore(true)
-	defer teardown()
-
-	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
-
-	t.Run("Successfully generates API key", func(t *testing.T) {
-		desc := "test-1"
-		rawKey, apiKey, err := service.GenerateApiKey(portainer.User{ID: 1}, desc)
-		is.NoError(err)
-		is.NotEmpty(rawKey)
-		is.NotEmpty(apiKey)
-		is.Equal(desc, apiKey.Description)
-	})
-
-	t.Run("Api key prefix is 7 chars", func(t *testing.T) {
-		rawKey, apiKey, err := service.GenerateApiKey(portainer.User{ID: 1}, "test-2")
-		is.NoError(err)
-
-		is.Equal(rawKey[:7], apiKey.Prefix)
-		is.Len(apiKey.Prefix, 7)
-	})
-
-	t.Run("Api key has 'ptr_' as prefix", func(t *testing.T) {
-		rawKey, _, err := service.GenerateApiKey(portainer.User{ID: 1}, "test-x")
-		is.NoError(err)
-
-		is.Equal(portainerAPIKeyPrefix, "ptr_")
-		is.True(strings.HasPrefix(rawKey, "ptr_"))
-	})
-
-	t.Run("Successfully caches API key", func(t *testing.T) {
-		user := portainer.User{ID: 1}
-		_, apiKey, err := service.GenerateApiKey(user, "test-3")
-		is.NoError(err)
-
-		userFromCache, apiKeyFromCache, ok := service.cache.Get(apiKey.Digest)
-		is.True(ok)
-		is.Equal(user, userFromCache)
-		is.Equal(apiKey, &apiKeyFromCache)
-	})
-
-	t.Run("Decoded raw api-key digest matches generated digest", func(t *testing.T) {
-		rawKey, apiKey, err := service.GenerateApiKey(portainer.User{ID: 1}, "test-4")
-		is.NoError(err)
-
-		generatedDigest := sha256.Sum256([]byte(rawKey))
-
-		is.Equal(apiKey.Digest, generatedDigest[:])
-	})
-}
-
-func Test_GetAPIKey(t *testing.T) {
-	is := assert.New(t)
-
-	_, store, teardown := datastore.MustNewTestStore(true)
-	defer teardown()
-
-	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
-
-	t.Run("Successfully returns all API keys", func(t *testing.T) {
-		user := portainer.User{ID: 1}
-		_, apiKey, err := service.GenerateApiKey(user, "test-1")
-		is.NoError(err)
-
-		apiKeyGot, err := service.GetAPIKey(apiKey.ID)
-		is.NoError(err)
-
-		is.Equal(apiKey, apiKeyGot)
-	})
-}
-
-func Test_GetAPIKeys(t *testing.T) {
-	is := assert.New(t)
-
-	_, store, teardown := datastore.MustNewTestStore(true)
-	defer teardown()
-
-	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
-
-	t.Run("Successfully returns all API keys", func(t *testing.T) {
-		user := portainer.User{ID: 1}
-		_, _, err := service.GenerateApiKey(user, "test-1")
-		is.NoError(err)
-		_, _, err = service.GenerateApiKey(user, "test-2")
-		is.NoError(err)
-
-		keys, err := service.GetAPIKeys(user.ID)
-		is.NoError(err)
-		is.Len(keys, 2)
-	})
-}
-
-func Test_GetDigestUserAndKey(t *testing.T) {
-	is := assert.New(t)
-
-	_, store, teardown := datastore.MustNewTestStore(true)
-	defer teardown()
-
-	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
-
-	t.Run("Successfully returns user and api key associated to digest", func(t *testing.T) {
-		user := portainer.User{ID: 1}
-		_, apiKey, err := service.GenerateApiKey(user, "test-1")
-		is.NoError(err)
-
-		userGot, apiKeyGot, err := service.GetDigestUserAndKey(apiKey.Digest)
-		is.NoError(err)
-		is.Equal(user, userGot)
-		is.Equal(*apiKey, apiKeyGot)
-	})
-
-	t.Run("Successfully caches user and api key associated to digest", func(t *testing.T) {
-		user := portainer.User{ID: 1}
-		_, apiKey, err := service.GenerateApiKey(user, "test-1")
-		is.NoError(err)
-
-		userGot, apiKeyGot, err := service.GetDigestUserAndKey(apiKey.Digest)
-		is.NoError(err)
-		is.Equal(user, userGot)
-		is.Equal(*apiKey, apiKeyGot)
-
-		userFromCache, apiKeyFromCache, ok := service.cache.Get(apiKey.Digest)
-		is.True(ok)
-		is.Equal(userGot, userFromCache)
-		is.Equal(apiKeyGot, apiKeyFromCache)
-	})
-}
-
-func Test_UpdateAPIKey(t *testing.T) {
-	is := assert.New(t)
-
-	_, store, teardown := datastore.MustNewTestStore(true)
-	defer teardown()
-
-	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
-
-	t.Run("Successfully updates the api-key LastUsed time", func(t *testing.T) {
-		user := portainer.User{ID: 1}
-		store.User().Create(&user)
-		_, apiKey, err := service.GenerateApiKey(user, "test-x")
-		is.NoError(err)
-
-		apiKey.LastUsed = time.Now().UTC().Unix()
-		err = service.UpdateAPIKey(apiKey)
-		is.NoError(err)
-
-		_, apiKeyGot, err := service.GetDigestUserAndKey(apiKey.Digest)
-		is.NoError(err)
-
-		log.Println(apiKey)
-		log.Println(apiKeyGot)
-
-		is.Equal(apiKey.LastUsed, apiKeyGot.LastUsed)
-
-	})
-
-	t.Run("Successfully updates api-key in cache upon api-key update", func(t *testing.T) {
-		_, apiKey, err := service.GenerateApiKey(portainer.User{ID: 1}, "test-x2")
-		is.NoError(err)
-
-		_, apiKeyFromCache, ok := service.cache.Get(apiKey.Digest)
-		is.True(ok)
-		is.Equal(*apiKey, apiKeyFromCache)
-
-		apiKey.LastUsed = time.Now().UTC().Unix()
-		is.NotEqual(*apiKey, apiKeyFromCache)
-
-		err = service.UpdateAPIKey(apiKey)
-		is.NoError(err)
-
-		_, updatedAPIKeyFromCache, ok := service.cache.Get(apiKey.Digest)
-		is.True(ok)
-		is.Equal(*apiKey, updatedAPIKeyFromCache)
-	})
-}
-
-func Test_DeleteAPIKey(t *testing.T) {
-	is := assert.New(t)
-
-	_, store, teardown := datastore.MustNewTestStore(true)
-	defer teardown()
-
-	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
-
-	t.Run("Successfully updates the api-key", func(t *testing.T) {
-		user := portainer.User{ID: 1}
-		_, apiKey, err := service.GenerateApiKey(user, "test-1")
-		is.NoError(err)
-
-		_, apiKeyGot, err := service.GetDigestUserAndKey(apiKey.Digest)
-		is.NoError(err)
-		is.Equal(*apiKey, apiKeyGot)
-
-		err = service.DeleteAPIKey(apiKey.ID)
-		is.NoError(err)
-
-		_, _, err = service.GetDigestUserAndKey(apiKey.Digest)
-		is.Error(err)
-	})
-
-	t.Run("Successfully removes api-key from cache upon deletion", func(t *testing.T) {
-		user := portainer.User{ID: 1}
-		_, apiKey, err := service.GenerateApiKey(user, "test-1")
-		is.NoError(err)
-
-		_, apiKeyFromCache, ok := service.cache.Get(apiKey.Digest)
-		is.True(ok)
-		is.Equal(*apiKey, apiKeyFromCache)
-
-		err = service.DeleteAPIKey(apiKey.ID)
-		is.NoError(err)
-
-		_, _, ok = service.cache.Get(apiKey.Digest)
-		is.False(ok)
-	})
-}
-
-func Test_InvalidateUserKeyCache(t *testing.T) {
-	is := assert.New(t)
-
-	_, store, teardown := datastore.MustNewTestStore(true)
-	defer teardown()
-
-	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
-
-	t.Run("Successfully updates evicts keys from cache", func(t *testing.T) {
-		// generate api keys
-		user := portainer.User{ID: 1}
-		_, apiKey1, err := service.GenerateApiKey(user, "test-1")
-		is.NoError(err)
-
-		_, apiKey2, err := service.GenerateApiKey(user, "test-2")
-		is.NoError(err)
-
-		// verify api keys are present in cache
-		_, apiKeyFromCache, ok := service.cache.Get(apiKey1.Digest)
-		is.True(ok)
-		is.Equal(*apiKey1, apiKeyFromCache)
-
-		_, apiKeyFromCache, ok = service.cache.Get(apiKey2.Digest)
-		is.True(ok)
-		is.Equal(*apiKey2, apiKeyFromCache)
-
-		// evict cache
-		ok = service.InvalidateUserKeyCache(user.ID)
-		is.True(ok)
-
-		// verify users keys have been flushed from cache
-		_, _, ok = service.cache.Get(apiKey1.Digest)
-		is.False(ok)
-
-		_, _, ok = service.cache.Get(apiKey2.Digest)
-		is.False(ok)
-	})
-
-	t.Run("User key eviction does not affect other users keys", func(t *testing.T) {
-		// generate keys for 2 users
-		user1 := portainer.User{ID: 1}
-		_, apiKey1, err := service.GenerateApiKey(user1, "test-1")
-		is.NoError(err)
-
-		user2 := portainer.User{ID: 2}
-		_, apiKey2, err := service.GenerateApiKey(user2, "test-2")
-		is.NoError(err)
-
-		// verify keys in cache
-		_, apiKeyFromCache, ok := service.cache.Get(apiKey1.Digest)
-		is.True(ok)
-		is.Equal(*apiKey1, apiKeyFromCache)
-
-		_, apiKeyFromCache, ok = service.cache.Get(apiKey2.Digest)
-		is.True(ok)
-		is.Equal(*apiKey2, apiKeyFromCache)
-
-		// evict key of single user from cache
-		ok = service.cache.InvalidateUserKeyCache(user1.ID)
-		is.True(ok)
-
-		// verify user1 key has been flushed from cache
-		_, _, ok = service.cache.Get(apiKey1.Digest)
-		is.False(ok)
-
-		// verify user2 key is still in cache
-		_, _, ok = service.cache.Get(apiKey2.Digest)
-		is.True(ok)
-	})
-}

api/aws/ecr/authorization_token.go

@@ -1,61 +0,0 @@
-package ecr
-
-import (
-	"context"
-	"encoding/base64"
-	"fmt"
-	"strings"
-	"time"
-)
-
-func (s *Service) GetEncodedAuthorizationToken() (token *string, expiry *time.Time, err error) {
-	getAuthorizationTokenOutput, err := s.client.GetAuthorizationToken(context.TODO(), nil)
-	if err != nil {
-		return
-	}
-
-	if len(getAuthorizationTokenOutput.AuthorizationData) == 0 {
-		err = fmt.Errorf("AuthorizationData is empty")
-		return
-	}
-
-	authData := getAuthorizationTokenOutput.AuthorizationData[0]
-
-	token = authData.AuthorizationToken
-	expiry = authData.ExpiresAt
-
-	return
-}
-
-func (s *Service) GetAuthorizationToken() (token *string, expiry *time.Time, err error) {
-	tokenEncodedStr, expiry, err := s.GetEncodedAuthorizationToken()
-	if err != nil {
-		return
-	}
-
-	tokenByte, err := base64.StdEncoding.DecodeString(*tokenEncodedStr)
-	if err != nil {
-		return
-	}
-	tokenStr := string(tokenByte)
-	token = &tokenStr
-
-	return
-}
-
-func (s *Service) ParseAuthorizationToken(token string) (username string, password string, err error) {
-	if len(token) == 0 {
-		return
-	}
-
-	splitToken := strings.Split(token, ":")
-	if len(splitToken) < 2 {
-		err = fmt.Errorf("invalid ECR authorization token")
-		return
-	}
-
-	username = splitToken[0]
-	password = splitToken[1]
-
-	return
-}

api/aws/ecr/ecr.go

@@ -1,32 +0,0 @@
-package ecr
-
-import (
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/service/ecr"
-)
-
-type (
-	Service struct {
-		accessKey string
-		secretKey string
-		region    string
-		client    *ecr.Client
-	}
-)
-
-func NewService(accessKey, secretKey, region string) *Service {
-	options := ecr.Options{
-		Region:      region,
-		Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
-	}
-
-	client := ecr.New(options)
-
-	return &Service{
-		accessKey: accessKey,
-		secretKey: secretKey,
-		region:    region,
-		client:    client,
-	}
-}

api/backup/backup.go

@@ -3,17 +3,15 @@ package backup
 import (
 	"fmt"
 	"os"
-	"path"
 	"path/filepath"
 	"time"
 
 	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
-	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/offlinegate"
-	"github.com/sirupsen/logrus"
 )
 
 const rwxr__r__ os.FileMode = 0744
@@ -32,7 +30,7 @@ var filesToBackup = []string{
 }
 
 // Creates a tar.gz system archive and encrypts it if password is not empty. Returns a path to the archive file.
-func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datastore dataservices.DataStore, filestorePath string) (string, error) {
+func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datastore portainer.DataStore, filestorePath string) (string, error) {
 	unlock := gate.Lock()
 	defer unlock()
 
@@ -41,18 +39,6 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto
 		return "", errors.Wrap(err, "Failed to create backup dir")
 	}
 
-	{
-		// new export
-		exportFilename := path.Join(backupDirPath, fmt.Sprintf("export-%d.json", time.Now().Unix()))
-
-		err := datastore.Export(exportFilename)
-		if err != nil {
-			logrus.WithError(err).Debugf("failed to export to %s", exportFilename)
-		} else {
-			logrus.Debugf("exported to %s", exportFilename)
-		}
-	}
-
 	if err := backupDb(backupDirPath, datastore); err != nil {
 		return "", errors.Wrap(err, "Failed to backup database")
 	}
@@ -79,7 +65,7 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto
 	return archivePath, nil
 }
 
-func backupDb(backupDirPath string, datastore dataservices.DataStore) error {
+func backupDb(backupDirPath string, datastore portainer.DataStore) error {
 	backupWriter, err := os.Create(filepath.Join(backupDirPath, "portainer.db"))
 	if err != nil {
 		return err

api/backup/restore.go

@@ -8,10 +8,9 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
-	"github.com/portainer/portainer/api/database/boltdb"
-	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/offlinegate"
 )
@@ -19,7 +18,7 @@ import (
 var filesToRestore = append(filesToBackup, "portainer.db")
 
 // Restores system state from backup archive, will trigger system shutdown, when finished.
-func RestoreArchive(archive io.Reader, password string, filestorePath string, gate *offlinegate.OfflineGate, datastore dataservices.DataStore, shutdownTrigger context.CancelFunc) error {
+func RestoreArchive(archive io.Reader, password string, filestorePath string, gate *offlinegate.OfflineGate, datastore portainer.DataStore, shutdownTrigger context.CancelFunc) error {
 	var err error
 	if password != "" {
 		archive, err = decrypt(archive, password)
@@ -66,20 +65,5 @@ func restoreFiles(srcDir string, destinationDir string) error {
 			return err
 		}
 	}
-
-	// TODO:  This is very boltdb module specific once again due to the filename.  Move to bolt module? Refactor for another day
-
-	// Prevent the possibility of having both databases.  Remove any default new instance
-	os.Remove(filepath.Join(destinationDir, boltdb.DatabaseFileName))
-	os.Remove(filepath.Join(destinationDir, boltdb.EncryptedDatabaseFileName))
-
-	// Now copy the database.  It'll be either portainer.db or portainer.edb
-
-	// Note: CopyPath does not return an error if the source file doesn't exist
-	err := filesystem.CopyPath(filepath.Join(srcDir, boltdb.EncryptedDatabaseFileName), destinationDir)
-	if err != nil {
-		return err
-	}
-
-	return filesystem.CopyPath(filepath.Join(srcDir, boltdb.DatabaseFileName), destinationDir)
+	return nil
 }

api/bolt/backup.go

@@ -1,4 +1,4 @@
-package datastore
+package bolt
 
 import (
 	"fmt"
@@ -6,18 +6,20 @@ import (
 	"path"
 	"time"
 
-	plog "github.com/portainer/portainer/api/datastore/log"
+	plog "github.com/portainer/portainer/api/bolt/log"
 )
 
 var backupDefaults = struct {
-	backupDir string
-	commonDir string
+	backupDir        string
+	commonDir        string
+	databaseFileName string
 }{
 	"backups",
 	"common",
+	databaseFileName,
 }
 
-var backupLog = plog.NewScopedLog("database, backup")
+var backupLog = plog.NewScopedLog("bolt, backup")
 
 //
 // Backup Helpers
@@ -35,11 +37,11 @@ func (store *Store) createBackupFolders() {
 }
 
 func (store *Store) databasePath() string {
-	return store.connection.GetDatabaseFilePath()
+	return path.Join(store.path, databaseFileName)
 }
 
 func (store *Store) commonBackupDir() string {
-	return path.Join(store.connection.GetStorePath(), backupDefaults.backupDir, backupDefaults.commonDir)
+	return path.Join(store.path, backupDefaults.backupDir, backupDefaults.commonDir)
 }
 
 func (store *Store) copyDBFile(from string, to string) error {
@@ -53,38 +55,24 @@ func (store *Store) copyDBFile(from string, to string) error {
 
 // BackupOptions provide a helper to inject backup options
 type BackupOptions struct {
-	Version        int // I can't find this used for anything other than a filename
+	Version        int
 	BackupDir      string
 	BackupFileName string
 	BackupPath     string
 }
 
-// getBackupRestoreOptions returns options to store db at common backup dir location; used by:
-// - db backup prior to version upgrade
-// - db rollback
-func getBackupRestoreOptions(backupDir string) *BackupOptions {
-	return &BackupOptions{
-		BackupDir:      backupDir, //connection.commonBackupDir(),
-		BackupFileName: beforePortainerVersionUpgradeBackup,
-	}
-}
-
 func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
 	if options == nil {
 		options = &BackupOptions{}
 	}
 	if options.Version == 0 {
-		version, err := store.version()
-		if err != nil {
-			version = 0
-		}
-		options.Version = version
+		options.Version, _ = store.version()
 	}
 	if options.BackupDir == "" {
 		options.BackupDir = store.commonBackupDir()
 	}
 	if options.BackupFileName == "" {
-		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
+		options.BackupFileName = fmt.Sprintf("%s.%s.%s", backupDefaults.databaseFileName, fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
 	}
 	if options.BackupPath == "" {
 		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
@@ -93,7 +81,7 @@ func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
 }
 
 // BackupWithOptions backup current database with options
-func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
+func (store *Store) BackupWithOptions(options *BackupOptions) (string, error) {
 	backupLog.Info("creating db backup")
 	store.createBackupFolders()
 
@@ -106,7 +94,7 @@ func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
 // Restore strategies:
 // - default: restore latest from current edition
 // - restore a specific
-func (store *Store) restoreWithOptions(options *BackupOptions) error {
+func (store *Store) RestoreWithOptions(options *BackupOptions) error {
 	options = store.setupOptions(options)
 
 	// Check if backup file exist before restoring
@@ -128,12 +116,11 @@ func (store *Store) restoreWithOptions(options *BackupOptions) error {
 		return err
 	}
 
-	_, err = store.Open()
-	return err
+	return store.Open()
 }
 
 // RemoveWithOptions removes backup database based on supplied options
-func (store *Store) removeWithOptions(options *BackupOptions) error {
+func (store *Store) RemoveWithOptions(options *BackupOptions) error {
 	backupLog.Info("Removing db backup")
 
 	options = store.setupOptions(options)

api/bolt/backup_test.go

@@ -1,20 +1,29 @@
-package datastore
+package bolt
 
 import (
 	"fmt"
 	"os"
 	"path"
+	"path/filepath"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
 )
 
+// isFileExist is helper function to check for file existence
+func isFileExist(path string) bool {
+	matches, err := filepath.Glob(path)
+	if err != nil {
+		return false
+	}
+	return len(matches) > 0
+}
+
 func TestCreateBackupFolders(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false)
+	store, teardown := MustNewTestStore(false)
 	defer teardown()
 
-	connection := store.GetConnection()
-	backupPath := path.Join(connection.GetStorePath(), backupDefaults.backupDir)
+	backupPath := path.Join(store.path, backupDefaults.backupDir)
 
 	if isFileExist(backupPath) {
 		t.Error("Expect backups folder to not exist")
@@ -27,39 +36,38 @@ func TestCreateBackupFolders(t *testing.T) {
 }
 
 func TestStoreCreation(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true)
+	store, teardown := MustNewTestStore(true)
 	defer teardown()
 
 	if store == nil {
 		t.Error("Expect to create a store")
 	}
 
-	if store.CheckCurrentEdition() != nil {
+	if store.edition() != portainer.PortainerCE {
 		t.Error("Expect to get CE Edition")
 	}
 }
 
 func TestBackup(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true)
-	connection := store.GetConnection()
+	store, teardown := MustNewTestStore(true)
 	defer teardown()
 
 	t.Run("Backup should create default db backup", func(t *testing.T) {
 		store.VersionService.StoreDBVersion(portainer.DBVersion)
-		store.backupWithOptions(nil)
+		store.BackupWithOptions(nil)
 
-		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%03d.*", portainer.DBVersion))
+		backupFileName := path.Join(store.path, "backups", "common", fmt.Sprintf("portainer.db.%03d.*", portainer.DBVersion))
 		if !isFileExist(backupFileName) {
 			t.Errorf("Expect backup file to be created %s", backupFileName)
 		}
 	})
 
 	t.Run("BackupWithOption should create a name specific backup at common path", func(t *testing.T) {
-		store.backupWithOptions(&BackupOptions{
+		store.BackupWithOptions(&BackupOptions{
 			BackupFileName: beforePortainerVersionUpgradeBackup,
 			BackupDir:      store.commonBackupDir(),
 		})
-		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", beforePortainerVersionUpgradeBackup)
+		backupFileName := path.Join(store.path, "backups", "common", beforePortainerVersionUpgradeBackup)
 		if !isFileExist(backupFileName) {
 			t.Errorf("Expect backup file to be created %s", backupFileName)
 		}
@@ -67,7 +75,7 @@ func TestBackup(t *testing.T) {
 }
 
 func TestRemoveWithOptions(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true)
+	store, teardown := MustNewTestStore(true)
 	defer teardown()
 
 	t.Run("successfully removes file if existent", func(t *testing.T) {
@@ -84,7 +92,7 @@ func TestRemoveWithOptions(t *testing.T) {
 		}
 		f.Close()
 
-		err = store.removeWithOptions(options)
+		err = store.RemoveWithOptions(options)
 		if err != nil {
 			t.Errorf("RemoveWithOptions should successfully remove file; err=%w", err)
 		}
@@ -100,7 +108,7 @@ func TestRemoveWithOptions(t *testing.T) {
 			BackupFileName: "test.txt",
 		}
 
-		err := store.removeWithOptions(options)
+		err := store.RemoveWithOptions(options)
 		if err == nil {
 			t.Error("RemoveWithOptions should fail for non-existent file")
 		}

api/bolt/customtemplate/customtemplate.go

@@ -0,0 +1,96 @@
+package customtemplate
+
+import (
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "customtemplates"
+)
+
+// Service represents a service for managing custom template data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// CustomTemplates return an array containing all the custom templates.
+func (service *Service) CustomTemplates() ([]portainer.CustomTemplate, error) {
+	var customTemplates = make([]portainer.CustomTemplate, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var customTemplate portainer.CustomTemplate
+			err := internal.UnmarshalObjectWithJsoniter(v, &customTemplate)
+			if err != nil {
+				return err
+			}
+			customTemplates = append(customTemplates, customTemplate)
+		}
+
+		return nil
+	})
+
+	return customTemplates, err
+}
+
+// CustomTemplate returns an custom template by ID.
+func (service *Service) CustomTemplate(ID portainer.CustomTemplateID) (*portainer.CustomTemplate, error) {
+	var customTemplate portainer.CustomTemplate
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &customTemplate)
+	if err != nil {
+		return nil, err
+	}
+
+	return &customTemplate, nil
+}
+
+// UpdateCustomTemplate updates an custom template.
+func (service *Service) UpdateCustomTemplate(ID portainer.CustomTemplateID, customTemplate *portainer.CustomTemplate) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, customTemplate)
+}
+
+// DeleteCustomTemplate deletes an custom template.
+func (service *Service) DeleteCustomTemplate(ID portainer.CustomTemplateID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}
+
+// CreateCustomTemplate assign an ID to a new custom template and saves it.
+func (service *Service) CreateCustomTemplate(customTemplate *portainer.CustomTemplate) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		data, err := internal.MarshalObject(customTemplate)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(customTemplate.ID)), data)
+	})
+}
+
+// GetNextIdentifier returns the next identifier for a custom template.
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.connection, BucketName)
+}

api/bolt/datastore.go

@@ -0,0 +1,154 @@
+package bolt
+
+import (
+	"io"
+	"path"
+	"time"
+
+	"github.com/portainer/portainer/api/bolt/helmuserrepository"
+
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/customtemplate"
+	"github.com/portainer/portainer/api/bolt/dockerhub"
+	"github.com/portainer/portainer/api/bolt/edgegroup"
+	"github.com/portainer/portainer/api/bolt/edgejob"
+	"github.com/portainer/portainer/api/bolt/edgestack"
+	"github.com/portainer/portainer/api/bolt/endpoint"
+	"github.com/portainer/portainer/api/bolt/endpointgroup"
+	"github.com/portainer/portainer/api/bolt/endpointrelation"
+	"github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/bolt/extension"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/registry"
+	"github.com/portainer/portainer/api/bolt/resourcecontrol"
+	"github.com/portainer/portainer/api/bolt/role"
+	"github.com/portainer/portainer/api/bolt/schedule"
+	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/ssl"
+	"github.com/portainer/portainer/api/bolt/stack"
+	"github.com/portainer/portainer/api/bolt/tag"
+	"github.com/portainer/portainer/api/bolt/team"
+	"github.com/portainer/portainer/api/bolt/teammembership"
+	"github.com/portainer/portainer/api/bolt/tunnelserver"
+	"github.com/portainer/portainer/api/bolt/user"
+	"github.com/portainer/portainer/api/bolt/version"
+	"github.com/portainer/portainer/api/bolt/webhook"
+)
+
+const (
+	databaseFileName = "portainer.db"
+)
+
+// Store defines the implementation of portainer.DataStore using
+// BoltDB as the storage system.
+type Store struct {
+	path                      string
+	connection                *internal.DbConnection
+	isNew                     bool
+	fileService               portainer.FileService
+	CustomTemplateService     *customtemplate.Service
+	DockerHubService          *dockerhub.Service
+	EdgeGroupService          *edgegroup.Service
+	EdgeJobService            *edgejob.Service
+	EdgeStackService          *edgestack.Service
+	EndpointGroupService      *endpointgroup.Service
+	EndpointService           *endpoint.Service
+	EndpointRelationService   *endpointrelation.Service
+	ExtensionService          *extension.Service
+	HelmUserRepositoryService *helmuserrepository.Service
+	RegistryService           *registry.Service
+	ResourceControlService    *resourcecontrol.Service
+	RoleService               *role.Service
+	ScheduleService           *schedule.Service
+	SettingsService           *settings.Service
+	SSLSettingsService        *ssl.Service
+	StackService              *stack.Service
+	TagService                *tag.Service
+	TeamMembershipService     *teammembership.Service
+	TeamService               *team.Service
+	TunnelServerService       *tunnelserver.Service
+	UserService               *user.Service
+	VersionService            *version.Service
+	WebhookService            *webhook.Service
+}
+
+func (store *Store) version() (int, error) {
+	version, err := store.VersionService.DBVersion()
+	if err == errors.ErrObjectNotFound {
+		version = 0
+	}
+	return version, err
+}
+
+func (store *Store) edition() portainer.SoftwareEdition {
+	edition, err := store.VersionService.Edition()
+	if err == errors.ErrObjectNotFound {
+		edition = portainer.PortainerCE
+	}
+	return edition
+}
+
+// NewStore initializes a new Store and the associated services
+func NewStore(storePath string, fileService portainer.FileService) *Store {
+	return &Store{
+		path:        storePath,
+		fileService: fileService,
+		isNew:       true,
+		connection:  &internal.DbConnection{},
+	}
+}
+
+// Open opens and initializes the BoltDB database.
+func (store *Store) Open() error {
+	databasePath := path.Join(store.path, databaseFileName)
+	db, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return err
+	}
+	store.connection.DB = db
+
+	err = store.initServices()
+	if err != nil {
+		return err
+	}
+
+	// if we have DBVersion in the database then ensure we flag this as NOT a new store
+	if _, err := store.VersionService.DBVersion(); err == nil {
+		store.isNew = false
+	}
+
+	return nil
+}
+
+// Close closes the BoltDB database.
+// Safe to being called multiple times.
+func (store *Store) Close() error {
+	if store.connection.DB != nil {
+		return store.connection.Close()
+	}
+	return nil
+}
+
+// IsNew returns true if the database was just created and false if it is re-using
+// existing data.
+func (store *Store) IsNew() bool {
+	return store.isNew
+}
+
+// BackupTo backs up db to a provided writer.
+// It does hot backup and doesn't block other database reads and writes
+func (store *Store) BackupTo(w io.Writer) error {
+	return store.connection.View(func(tx *bolt.Tx) error {
+		_, err := tx.WriteTo(w)
+		return err
+	})
+}
+
+// CheckCurrentEdition checks if current edition is community edition
+func (store *Store) CheckCurrentEdition() error {
+	if store.edition() != portainer.PortainerCE {
+		return errors.ErrWrongDBEdition
+	}
+	return nil
+}

api/bolt/dockerhub/dockerhub.go

@@ -2,6 +2,7 @@ package dockerhub
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 )
 
 const (
@@ -12,16 +13,12 @@ const (
 
 // Service represents a service for managing Dockerhub data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection portainer.Connection) (*Service, error) {
-	err := connection.SetServiceName(BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
@@ -35,7 +32,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
 func (service *Service) DockerHub() (*portainer.DockerHub, error) {
 	var dockerhub portainer.DockerHub
 
-	err := service.connection.GetObject(BucketName, []byte(dockerHubKey), &dockerhub)
+	err := internal.GetObject(service.connection, BucketName, []byte(dockerHubKey), &dockerhub)
 	if err != nil {
 		return nil, err
 	}
@@ -45,5 +42,5 @@ func (service *Service) DockerHub() (*portainer.DockerHub, error) {
 
 // UpdateDockerHub updates a DockerHub object.
 func (service *Service) UpdateDockerHub(dockerhub *portainer.DockerHub) error {
-	return service.connection.UpdateObject(BucketName, []byte(dockerHubKey), dockerhub)
+	return internal.UpdateObject(service.connection, BucketName, []byte(dockerHubKey), dockerhub)
 }

api/bolt/edgegroup/edgegroup.go

@@ -0,0 +1,94 @@
+package edgegroup
+
+import (
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "edgegroups"
+)
+
+// Service represents a service for managing Edge group data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// EdgeGroups return an array containing all the Edge groups.
+func (service *Service) EdgeGroups() ([]portainer.EdgeGroup, error) {
+	var groups = make([]portainer.EdgeGroup, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var group portainer.EdgeGroup
+			err := internal.UnmarshalObjectWithJsoniter(v, &group)
+			if err != nil {
+				return err
+			}
+			groups = append(groups, group)
+		}
+
+		return nil
+	})
+
+	return groups, err
+}
+
+// EdgeGroup returns an Edge group by ID.
+func (service *Service) EdgeGroup(ID portainer.EdgeGroupID) (*portainer.EdgeGroup, error) {
+	var group portainer.EdgeGroup
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &group)
+	if err != nil {
+		return nil, err
+	}
+
+	return &group, nil
+}
+
+// UpdateEdgeGroup updates an Edge group.
+func (service *Service) UpdateEdgeGroup(ID portainer.EdgeGroupID, group *portainer.EdgeGroup) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, group)
+}
+
+// DeleteEdgeGroup deletes an Edge group.
+func (service *Service) DeleteEdgeGroup(ID portainer.EdgeGroupID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}
+
+// CreateEdgeGroup assign an ID to a new Edge group and saves it.
+func (service *Service) CreateEdgeGroup(group *portainer.EdgeGroup) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		group.ID = portainer.EdgeGroupID(id)
+
+		data, err := internal.MarshalObject(group)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(group.ID)), data)
+	})
+}

api/bolt/edgejob/edgejob.go

@@ -0,0 +1,101 @@
+package edgejob
+
+import (
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "edgejobs"
+)
+
+// Service represents a service for managing edge jobs data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// EdgeJobs returns a list of Edge jobs
+func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
+	var edgeJobs = make([]portainer.EdgeJob, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var edgeJob portainer.EdgeJob
+			err := internal.UnmarshalObject(v, &edgeJob)
+			if err != nil {
+				return err
+			}
+			edgeJobs = append(edgeJobs, edgeJob)
+		}
+
+		return nil
+	})
+
+	return edgeJobs, err
+}
+
+// EdgeJob returns an Edge job by ID
+func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error) {
+	var edgeJob portainer.EdgeJob
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &edgeJob)
+	if err != nil {
+		return nil, err
+	}
+
+	return &edgeJob, nil
+}
+
+// CreateEdgeJob creates a new Edge job
+func (service *Service) CreateEdgeJob(edgeJob *portainer.EdgeJob) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		if edgeJob.ID == 0 {
+			id, _ := bucket.NextSequence()
+			edgeJob.ID = portainer.EdgeJobID(id)
+		}
+
+		data, err := internal.MarshalObject(edgeJob)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(edgeJob.ID)), data)
+	})
+}
+
+// UpdateEdgeJob updates an Edge job by ID
+func (service *Service) UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, edgeJob)
+}
+
+// DeleteEdgeJob deletes an Edge job
+func (service *Service) DeleteEdgeJob(ID portainer.EdgeJobID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}
+
+// GetNextIdentifier returns the next identifier for an environment(endpoint).
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.connection, BucketName)
+}

api/bolt/edgestack/edgestack.go

@@ -0,0 +1,101 @@
+package edgestack
+
+import (
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "edge_stack"
+)
+
+// Service represents a service for managing Edge stack data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// EdgeStacks returns an array containing all edge stacks
+func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
+	var stacks = make([]portainer.EdgeStack, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var stack portainer.EdgeStack
+			err := internal.UnmarshalObject(v, &stack)
+			if err != nil {
+				return err
+			}
+			stacks = append(stacks, stack)
+		}
+
+		return nil
+	})
+
+	return stacks, err
+}
+
+// EdgeStack returns an Edge stack by ID.
+func (service *Service) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStack, error) {
+	var stack portainer.EdgeStack
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &stack)
+	if err != nil {
+		return nil, err
+	}
+
+	return &stack, nil
+}
+
+// CreateEdgeStack assign an ID to a new Edge stack and saves it.
+func (service *Service) CreateEdgeStack(edgeStack *portainer.EdgeStack) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		if edgeStack.ID == 0 {
+			id, _ := bucket.NextSequence()
+			edgeStack.ID = portainer.EdgeStackID(id)
+		}
+
+		data, err := internal.MarshalObject(edgeStack)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(edgeStack.ID)), data)
+	})
+}
+
+// UpdateEdgeStack updates an Edge stack.
+func (service *Service) UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, edgeStack)
+}
+
+// DeleteEdgeStack deletes an Edge stack.
+func (service *Service) DeleteEdgeStack(ID portainer.EdgeStackID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}
+
+// GetNextIdentifier returns the next identifier for an environment(endpoint).
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.connection, BucketName)
+}

api/bolt/endpoint/endpoint.go

@@ -0,0 +1,145 @@
+package endpoint
+
+import (
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "endpoints"
+)
+
+// Service represents a service for managing environment(endpoint) data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// Endpoint returns an environment(endpoint) by ID.
+func (service *Service) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error) {
+	var endpoint portainer.Endpoint
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return &endpoint, nil
+}
+
+// UpdateEndpoint updates an environment(endpoint).
+func (service *Service) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, endpoint)
+}
+
+// DeleteEndpoint deletes an environment(endpoint).
+func (service *Service) DeleteEndpoint(ID portainer.EndpointID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}
+
+// Endpoints return an array containing all the environments(endpoints).
+func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
+	var endpoints = make([]portainer.Endpoint, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var endpoint portainer.Endpoint
+			err := internal.UnmarshalObjectWithJsoniter(v, &endpoint)
+			if err != nil {
+				return err
+			}
+			endpoints = append(endpoints, endpoint)
+		}
+
+		return nil
+	})
+
+	return endpoints, err
+}
+
+// CreateEndpoint assign an ID to a new environment(endpoint) and saves it.
+func (service *Service) CreateEndpoint(endpoint *portainer.Endpoint) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		// We manually manage sequences for environments(endpoints)
+		err := bucket.SetSequence(uint64(endpoint.ID))
+		if err != nil {
+			return err
+		}
+
+		data, err := internal.MarshalObject(endpoint)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(endpoint.ID)), data)
+	})
+}
+
+// GetNextIdentifier returns the next identifier for an environment(endpoint).
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.connection, BucketName)
+}
+
+// Synchronize creates, updates and deletes environments(endpoints) inside a single transaction.
+func (service *Service) Synchronize(toCreate, toUpdate, toDelete []*portainer.Endpoint) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		for _, endpoint := range toCreate {
+			id, _ := bucket.NextSequence()
+			endpoint.ID = portainer.EndpointID(id)
+
+			data, err := internal.MarshalObject(endpoint)
+			if err != nil {
+				return err
+			}
+
+			err = bucket.Put(internal.Itob(int(endpoint.ID)), data)
+			if err != nil {
+				return err
+			}
+		}
+
+		for _, endpoint := range toUpdate {
+			data, err := internal.MarshalObject(endpoint)
+			if err != nil {
+				return err
+			}
+
+			err = bucket.Put(internal.Itob(int(endpoint.ID)), data)
+			if err != nil {
+				return err
+			}
+		}
+
+		for _, endpoint := range toDelete {
+			err := bucket.Delete(internal.Itob(int(endpoint.ID)))
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}

api/bolt/endpointgroup/endpointgroup.go

@@ -0,0 +1,95 @@
+package endpointgroup
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "endpoint_groups"
+)
+
+// Service represents a service for managing environment(endpoint) data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// EndpointGroup returns an environment(endpoint) group by ID.
+func (service *Service) EndpointGroup(ID portainer.EndpointGroupID) (*portainer.EndpointGroup, error) {
+	var endpointGroup portainer.EndpointGroup
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &endpointGroup)
+	if err != nil {
+		return nil, err
+	}
+
+	return &endpointGroup, nil
+}
+
+// UpdateEndpointGroup updates an environment(endpoint) group.
+func (service *Service) UpdateEndpointGroup(ID portainer.EndpointGroupID, endpointGroup *portainer.EndpointGroup) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, endpointGroup)
+}
+
+// DeleteEndpointGroup deletes an environment(endpoint) group.
+func (service *Service) DeleteEndpointGroup(ID portainer.EndpointGroupID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}
+
+// EndpointGroups return an array containing all the environment(endpoint) groups.
+func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
+	var endpointGroups = make([]portainer.EndpointGroup, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var endpointGroup portainer.EndpointGroup
+			err := internal.UnmarshalObject(v, &endpointGroup)
+			if err != nil {
+				return err
+			}
+			endpointGroups = append(endpointGroups, endpointGroup)
+		}
+
+		return nil
+	})
+
+	return endpointGroups, err
+}
+
+// CreateEndpointGroup assign an ID to a new environment(endpoint) group and saves it.
+func (service *Service) CreateEndpointGroup(endpointGroup *portainer.EndpointGroup) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		endpointGroup.ID = portainer.EndpointGroupID(id)
+
+		data, err := internal.MarshalObject(endpointGroup)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(endpointGroup.ID)), data)
+	})
+}

api/bolt/endpointrelation/endpointrelation.go

@@ -0,0 +1,68 @@
+package endpointrelation
+
+import (
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "endpoint_relations"
+)
+
+// Service represents a service for managing environment(endpoint) relation data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// EndpointRelation returns a Environment(Endpoint) relation object by EndpointID
+func (service *Service) EndpointRelation(endpointID portainer.EndpointID) (*portainer.EndpointRelation, error) {
+	var endpointRelation portainer.EndpointRelation
+	identifier := internal.Itob(int(endpointID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &endpointRelation)
+	if err != nil {
+		return nil, err
+	}
+
+	return &endpointRelation, nil
+}
+
+// CreateEndpointRelation saves endpointRelation
+func (service *Service) CreateEndpointRelation(endpointRelation *portainer.EndpointRelation) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		data, err := internal.MarshalObject(endpointRelation)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(endpointRelation.EndpointID)), data)
+	})
+}
+
+// UpdateEndpointRelation updates an Environment(Endpoint) relation object
+func (service *Service) UpdateEndpointRelation(EndpointID portainer.EndpointID, endpointRelation *portainer.EndpointRelation) error {
+	identifier := internal.Itob(int(EndpointID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, endpointRelation)
+}
+
+// DeleteEndpointRelation deletes an Environment(Endpoint) relation object
+func (service *Service) DeleteEndpointRelation(EndpointID portainer.EndpointID) error {
+	identifier := internal.Itob(int(EndpointID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}

api/bolt/errors/errors.go

@@ -0,0 +1,8 @@
+package errors
+
+import "errors"
+
+var (
+	ErrObjectNotFound = errors.New("Object not found inside the database")
+	ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
+)

api/bolt/extension/extension.go

@@ -1,10 +1,10 @@
 package extension
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
 )
 
 const (
@@ -14,16 +14,12 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection portainer.Connection) (*Service, error) {
-	err := connection.SetServiceName(BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
@@ -36,9 +32,9 @@ func NewService(connection portainer.Connection) (*Service, error) {
 // Extension returns a extension by ID
 func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extension, error) {
 	var extension portainer.Extension
-	identifier := service.connection.ConvertToKey(int(ID))
+	identifier := internal.Itob(int(ID))
 
-	err := service.connection.GetObject(BucketName, identifier, &extension)
+	err := internal.GetObject(service.connection, BucketName, identifier, &extension)
 	if err != nil {
 		return nil, err
 	}
@@ -50,29 +46,41 @@ func (service *Service) Extension(ID portainer.ExtensionID) (*portainer.Extensio
 func (service *Service) Extensions() ([]portainer.Extension, error) {
 	var extensions = make([]portainer.Extension, 0)
 
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.Extension{},
-		func(obj interface{}) (interface{}, error) {
-			extension, ok := obj.(*portainer.Extension)
-			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Extension object")
-				return nil, fmt.Errorf("Failed to convert to Extension object: %s", obj)
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var extension portainer.Extension
+			err := internal.UnmarshalObject(v, &extension)
+			if err != nil {
+				return err
 			}
-			extensions = append(extensions, *extension)
-			return &portainer.Extension{}, nil
-		})
+			extensions = append(extensions, extension)
+		}
+
+		return nil
+	})
 
 	return extensions, err
 }
 
 // Persist persists a extension inside the database.
 func (service *Service) Persist(extension *portainer.Extension) error {
-	return service.connection.CreateObjectWithId(BucketName, int(extension.ID), extension)
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		data, err := internal.MarshalObject(extension)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(extension.ID)), data)
+	})
 }
 
 // DeleteExtension deletes a Extension.
 func (service *Service) DeleteExtension(ID portainer.ExtensionID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/helmuserrepository/helmuserrepository.go

@@ -0,0 +1,73 @@
+package helmuserrepository
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "helm_user_repository"
+)
+
+// Service represents a service for managing environment(endpoint) data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// HelmUserRepositoryByUserID return an array containing all the HelmUserRepository objects where the specified userID is present.
+func (service *Service) HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error) {
+	var result = make([]portainer.HelmUserRepository, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var record portainer.HelmUserRepository
+			err := internal.UnmarshalObject(v, &record)
+			if err != nil {
+				return err
+			}
+
+			if record.UserID == userID {
+				result = append(result, record)
+			}
+		}
+
+		return nil
+	})
+
+	return result, err
+}
+
+// CreateHelmUserRepository creates a new HelmUserRepository object.
+func (service *Service) CreateHelmUserRepository(record *portainer.HelmUserRepository) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		record.ID = portainer.HelmUserRepositoryID(id)
+
+		data, err := internal.MarshalObject(record)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(record.ID)), data)
+	})
+}

api/bolt/init.go

@@ -1,55 +1,32 @@
-package datastore
+package bolt
 
 import (
 	"github.com/gofrs/uuid"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
 )
 
 // Init creates the default data set.
 func (store *Store) Init() error {
-	err := store.checkOrCreateInstanceID()
-	if err != nil {
-		return err
-	}
-
-	err = store.checkOrCreateDefaultSettings()
-	if err != nil {
-		return err
-	}
-
-	err = store.checkOrCreateDefaultSSLSettings()
-	if err != nil {
-		return err
-	}
-
-	err = store.checkOrCreateDefaultData()
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (store *Store) checkOrCreateInstanceID() error {
 	instanceID, err := store.VersionService.InstanceID()
-	if store.IsErrObjectNotFound(err) {
+	if err == errors.ErrObjectNotFound {
 		uid, err := uuid.NewV4()
 		if err != nil {
 			return err
 		}
 
 		instanceID = uid.String()
-		return store.VersionService.StoreInstanceID(instanceID)
+		err = store.VersionService.StoreInstanceID(instanceID)
+		if err != nil {
+			return err
+		}
+	} else if err != nil {
+		return err
 	}
-	return err
-}
 
-func (store *Store) checkOrCreateDefaultSettings() error {
-	// TODO: these need to also be applied when importing
-	settings, err := store.SettingsService.Settings()
-	if store.IsErrObjectNotFound(err) {
+	_, err = store.SettingsService.Settings()
+	if err == errors.ErrObjectNotFound {
 		defaultSettings := &portainer.Settings{
-			EnableTelemetry:      true,
 			AuthenticationMethod: portainer.AuthenticationInternal,
 			BlackListedLabels:    make([]portainer.Pair, 0),
 			LDAPSettings: portainer.LDAPSettings{
@@ -57,16 +34,14 @@ func (store *Store) checkOrCreateDefaultSettings() error {
 				AutoCreateUsers: true,
 				TLSConfig:       portainer.TLSConfiguration{},
 				SearchSettings: []portainer.LDAPSearchSettings{
-					{},
+					portainer.LDAPSearchSettings{},
 				},
 				GroupSearchSettings: []portainer.LDAPGroupSearchSettings{
-					{},
+					portainer.LDAPGroupSearchSettings{},
 				},
 			},
-			OAuthSettings: portainer.OAuthSettings{
-				SSO: true,
-			},
-			SnapshotInterval:         portainer.DefaultSnapshotInterval,
+			OAuthSettings: portainer.OAuthSettings{},
+
 			EdgeAgentCheckinInterval: portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
 			TemplatesURL:             portainer.DefaultTemplatesURL,
 			HelmRepositoryURL:        portainer.DefaultHelmRepositoryURL,
@@ -75,33 +50,30 @@ func (store *Store) checkOrCreateDefaultSettings() error {
 			KubectlShellImage:        portainer.DefaultKubectlShellImage,
 		}
 
-		return store.SettingsService.UpdateSettings(defaultSettings)
-	}
-	if err != nil {
+		err = store.SettingsService.UpdateSettings(defaultSettings)
+		if err != nil {
+			return err
+		}
+	} else if err != nil {
 		return err
 	}
 
-	if settings.UserSessionTimeout == "" {
-		settings.UserSessionTimeout = portainer.DefaultUserSessionTimeout
-		return store.Settings().UpdateSettings(settings)
-	}
-	return nil
-}
+	_, err = store.SSLSettings().Settings()
+	if err != nil {
+		if err != errors.ErrObjectNotFound {
+			return err
+		}
 
-func (store *Store) checkOrCreateDefaultSSLSettings() error {
-	_, err := store.SSLSettings().Settings()
-
-	if store.IsErrObjectNotFound(err) {
 		defaultSSLSettings := &portainer.SSLSettings{
 			HTTPEnabled: true,
 		}
 
-		return store.SSLSettings().UpdateSettings(defaultSSLSettings)
+		err = store.SSLSettings().UpdateSettings(defaultSSLSettings)
+		if err != nil {
+			return err
+		}
 	}
-	return err
-}
 
-func (store *Store) checkOrCreateDefaultData() error {
 	groups, err := store.EndpointGroupService.EndpointGroups()
 	if err != nil {
 		return err
@@ -117,10 +89,11 @@ func (store *Store) checkOrCreateDefaultData() error {
 			TagIDs:             []portainer.TagID{},
 		}
 
-		err = store.EndpointGroupService.Create(unassignedGroup)
+		err = store.EndpointGroupService.CreateEndpointGroup(unassignedGroup)
 		if err != nil {
 			return err
 		}
 	}
+
 	return nil
 }

api/bolt/internal/db.go

@@ -0,0 +1,100 @@
+package internal
+
+import (
+	"encoding/binary"
+
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api/bolt/errors"
+)
+
+type DbConnection struct {
+	*bolt.DB
+}
+
+// Itob returns an 8-byte big endian representation of v.
+// This function is typically used for encoding integer IDs to byte slices
+// so that they can be used as BoltDB keys.
+func Itob(v int) []byte {
+	b := make([]byte, 8)
+	binary.BigEndian.PutUint64(b, uint64(v))
+	return b
+}
+
+// CreateBucket is a generic function used to create a bucket inside a bolt database.
+func CreateBucket(connection *DbConnection, bucketName string) error {
+	return connection.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}
+
+// GetObject is a generic function used to retrieve an unmarshalled object from a bolt database.
+func GetObject(connection *DbConnection, bucketName string, key []byte, object interface{}) error {
+	var data []byte
+
+	err := connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(bucketName))
+
+		value := bucket.Get(key)
+		if value == nil {
+			return errors.ErrObjectNotFound
+		}
+
+		data = make([]byte, len(value))
+		copy(data, value)
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return UnmarshalObject(data, object)
+}
+
+// UpdateObject is a generic function used to update an object inside a bolt database.
+func UpdateObject(connection *DbConnection, bucketName string, key []byte, object interface{}) error {
+	return connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(bucketName))
+
+		data, err := MarshalObject(object)
+		if err != nil {
+			return err
+		}
+
+		err = bucket.Put(key, data)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+// DeleteObject is a generic function used to delete an object inside a bolt database.
+func DeleteObject(connection *DbConnection, bucketName string, key []byte) error {
+	return connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(bucketName))
+		return bucket.Delete(key)
+	})
+}
+
+// GetNextIdentifier is a generic function that returns the specified bucket identifier incremented by 1.
+func GetNextIdentifier(connection *DbConnection, bucketName string) int {
+	var identifier int
+
+	connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(bucketName))
+		id, err := bucket.NextSequence()
+		if err != nil {
+			return err
+		}
+		identifier = int(id)
+		return nil
+	})
+
+	return identifier
+}

api/bolt/internal/json.go

@@ -0,0 +1,25 @@
+package internal
+
+import (
+	"encoding/json"
+
+	jsoniter "github.com/json-iterator/go"
+)
+
+// MarshalObject encodes an object to binary format
+func MarshalObject(object interface{}) ([]byte, error) {
+	return json.Marshal(object)
+}
+
+// UnmarshalObject decodes an object from binary data
+func UnmarshalObject(data []byte, object interface{}) error {
+	return json.Unmarshal(data, object)
+}
+
+// UnmarshalObjectWithJsoniter decodes an object from binary data
+// using the jsoniter library. It is mainly used to accelerate environment(endpoint)
+// decoding at the moment.
+func UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
+	var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
+	return jsoni.Unmarshal(data, &object)
+}

api/bolt/log/log.test.go

@@ -0,0 +1 @@
+package log

api/bolt/migrate_data.go

@@ -1,61 +1,28 @@
-package datastore
+package bolt
 
 import (
 	"fmt"
-	"runtime/debug"
 
 	"github.com/portainer/portainer/api/cli"
-	"github.com/portainer/portainer/api/dataservices/errors"
-	plog "github.com/portainer/portainer/api/datastore/log"
-	"github.com/portainer/portainer/api/datastore/migrator"
-	"github.com/portainer/portainer/api/internal/authorization"
 
 	werrors "github.com/pkg/errors"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+	plog "github.com/portainer/portainer/api/bolt/log"
+	"github.com/portainer/portainer/api/bolt/migrator"
+	"github.com/portainer/portainer/api/internal/authorization"
 )
 
 const beforePortainerVersionUpgradeBackup = "portainer.db.bak"
 
-var migrateLog = plog.NewScopedLog("database, migrate")
-
-func (store *Store) MigrateData() error {
-	version, err := store.version()
-	if err != nil {
-		return err
-	}
-
-	migratorParams := &migrator.MigratorParameters{
-		DatabaseVersion:         version,
-		EndpointGroupService:    store.EndpointGroupService,
-		EndpointService:         store.EndpointService,
-		EndpointRelationService: store.EndpointRelationService,
-		ExtensionService:        store.ExtensionService,
-		FDOProfilesService:      store.FDOProfilesService,
-		RegistryService:         store.RegistryService,
-		ResourceControlService:  store.ResourceControlService,
-		RoleService:             store.RoleService,
-		ScheduleService:         store.ScheduleService,
-		SettingsService:         store.SettingsService,
-		StackService:            store.StackService,
-		TagService:              store.TagService,
-		TeamMembershipService:   store.TeamMembershipService,
-		UserService:             store.UserService,
-		VersionService:          store.VersionService,
-		FileService:             store.fileService,
-		DockerhubService:        store.DockerHubService,
-		AuthorizationService:    authorization.NewService(store),
-	}
-
-	return store.connectionMigrateData(migratorParams)
-}
+var migrateLog = plog.NewScopedLog("bolt, migrate")
 
 // FailSafeMigrate backup and restore DB if migration fail
 func (store *Store) FailSafeMigrate(migrator *migrator.Migrator) (err error) {
 	defer func() {
 		if e := recover(); e != nil {
 			store.Rollback(true)
-			// return error with cause and stacktrace (recover() doesn't include a stacktrace)
-			err = fmt.Errorf("%v %s", e, string(debug.Stack()))
+			err = fmt.Errorf("%v", e)
 		}
 	}()
 
@@ -67,11 +34,18 @@ func (store *Store) FailSafeMigrate(migrator *migrator.Migrator) (err error) {
 // MigrateData automatically migrate the data based on the DBVersion.
 // This process is only triggered on an existing database, not if the database was just created.
 // if force is true, then migrate regardless.
-func (store *Store) connectionMigrateData(migratorParams *migrator.MigratorParameters) error {
-	migrator := migrator.NewMigrator(migratorParams)
+func (store *Store) MigrateData(force bool) error {
+	if store.isNew && !force {
+		return store.VersionService.StoreDBVersion(portainer.DBVersion)
+	}
+
+	migrator, err := store.newMigrator()
+	if err != nil {
+		return err
+	}
 
 	// backup db file before upgrading DB to support rollback
-	isUpdating, err := migratorParams.VersionService.IsUpdating()
+	isUpdating, err := store.VersionService.IsUpdating()
 	if err != nil && err != errors.ErrObjectNotFound {
 		return err
 	}
@@ -95,16 +69,56 @@ func (store *Store) connectionMigrateData(migratorParams *migrator.MigratorParam
 	return nil
 }
 
+func (store *Store) newMigrator() (*migrator.Migrator, error) {
+	version, err := store.version()
+	if err != nil {
+		return nil, err
+	}
+
+	migratorParams := &migrator.Parameters{
+		DB:                      store.connection.DB,
+		DatabaseVersion:         version,
+		EndpointGroupService:    store.EndpointGroupService,
+		EndpointService:         store.EndpointService,
+		EndpointRelationService: store.EndpointRelationService,
+		ExtensionService:        store.ExtensionService,
+		RegistryService:         store.RegistryService,
+		ResourceControlService:  store.ResourceControlService,
+		RoleService:             store.RoleService,
+		ScheduleService:         store.ScheduleService,
+		SettingsService:         store.SettingsService,
+		StackService:            store.StackService,
+		TagService:              store.TagService,
+		TeamMembershipService:   store.TeamMembershipService,
+		UserService:             store.UserService,
+		VersionService:          store.VersionService,
+		FileService:             store.fileService,
+		DockerhubService:        store.DockerHubService,
+		AuthorizationService:    authorization.NewService(store),
+	}
+	return migrator.NewMigrator(migratorParams), nil
+}
+
+// getBackupRestoreOptions returns options to store db at common backup dir location; used by:
+// - db backup prior to version upgrade
+// - db rollback
+func getBackupRestoreOptions(store *Store) *BackupOptions {
+	return &BackupOptions{
+		BackupDir:      store.commonBackupDir(),
+		BackupFileName: beforePortainerVersionUpgradeBackup,
+	}
+}
+
 // backupVersion will backup the database or panic if any errors occur
 func (store *Store) backupVersion(migrator *migrator.Migrator) error {
 	migrateLog.Info("Backing up database prior to version upgrade...")
 
-	options := getBackupRestoreOptions(store.commonBackupDir())
+	options := getBackupRestoreOptions(store)
 
-	_, err := store.backupWithOptions(options)
+	_, err := store.BackupWithOptions(options)
 	if err != nil {
 		migrateLog.Error("An error occurred during database backup", err)
-		removalErr := store.removeWithOptions(options)
+		removalErr := store.RemoveWithOptions(options)
 		if removalErr != nil {
 			migrateLog.Error("An error occurred during store removal prior to backup", err)
 		}
@@ -115,7 +129,7 @@ func (store *Store) backupVersion(migrator *migrator.Migrator) error {
 }
 
 // Rollback to a pre-upgrade backup copy/snapshot of portainer.db
-func (store *Store) connectionRollback(force bool) error {
+func (store *Store) Rollback(force bool) error {
 
 	if !force {
 		confirmed, err := cli.Confirm("Are you sure you want to rollback your database to the previous backup?")
@@ -124,12 +138,12 @@ func (store *Store) connectionRollback(force bool) error {
 		}
 	}
 
-	options := getBackupRestoreOptions(store.commonBackupDir())
+	options := getBackupRestoreOptions(store)
 
-	err := store.restoreWithOptions(options)
+	err := store.RestoreWithOptions(options)
 	if err != nil {
 		return err
 	}
 
-	return store.connection.Close()
+	return store.Close()
 }

api/bolt/migrate_data_test.go

@@ -1,9 +1,8 @@
-package datastore
+package bolt
 
 import (
 	"fmt"
 	"log"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -12,32 +11,27 @@ import (
 
 // testVersion is a helper which tests current store version against wanted version
 func testVersion(store *Store, versionWant int, t *testing.T) {
-	v, err := store.VersionService.DBVersion()
-	if err != nil {
-		t.Errorf("Expect store version to be %d but was %d with error: %s", versionWant, v, err)
-	}
-	if v != versionWant {
+	if v, _ := store.version(); v != versionWant {
 		t.Errorf("Expect store version to be %d but was %d", versionWant, v)
 	}
 }
 
 func TestMigrateData(t *testing.T) {
 	t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
-		newStore, store, teardown := MustNewTestStore(false)
+		store, teardown := MustNewTestStore(false)
 		defer teardown()
 
-		if !newStore {
+		if !store.IsNew() {
 			t.Error("Expect a new DB")
 		}
 
-		// not called for new stores
-		//store.MigrateData()
+		store.MigrateData(false)
 
 		testVersion(store, portainer.DBVersion, t)
 		store.Close()
 
-		newStore, _ = store.Open()
-		if newStore {
+		store.Open()
+		if store.IsNew() {
 			t.Error("Expect store to NOT be new DB")
 		}
 	})
@@ -46,24 +40,24 @@ func TestMigrateData(t *testing.T) {
 		version         int
 		expectedVersion int
 	}{
-		{version: 17, expectedVersion: portainer.DBVersion},
+		{version: 2, expectedVersion: portainer.DBVersion},
 		{version: 21, expectedVersion: portainer.DBVersion},
 	}
 	for _, tc := range tests {
-		_, store, teardown := MustNewTestStore(true)
+		store, teardown := MustNewTestStore(true)
 		defer teardown()
 
 		// Setup data
 		store.VersionService.StoreDBVersion(tc.version)
 
 		// Required roles by migrations 22.2
-		store.RoleService.Create(&portainer.Role{ID: 1})
-		store.RoleService.Create(&portainer.Role{ID: 2})
-		store.RoleService.Create(&portainer.Role{ID: 3})
-		store.RoleService.Create(&portainer.Role{ID: 4})
+		store.RoleService.CreateRole(&portainer.Role{ID: 1})
+		store.RoleService.CreateRole(&portainer.Role{ID: 2})
+		store.RoleService.CreateRole(&portainer.Role{ID: 3})
+		store.RoleService.CreateRole(&portainer.Role{ID: 4})
 
 		t.Run(fmt.Sprintf("MigrateData for version %d", tc.version), func(t *testing.T) {
-			store.MigrateData()
+			store.MigrateData(true)
 			testVersion(store, tc.expectedVersion, t)
 		})
 
@@ -75,25 +69,25 @@ func TestMigrateData(t *testing.T) {
 	}
 
 	t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false)
+		store, teardown := MustNewTestStore(false)
 		defer teardown()
 
-		version := 17
+		version := 2
 		store.VersionService.StoreDBVersion(version)
 
-		store.MigrateData()
+		store.MigrateData(true)
 
 		testVersion(store, version, t)
 	})
 
 	t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false)
+		store, teardown := MustNewTestStore(false)
 		defer teardown()
 		store.VersionService.StoreDBVersion(0)
 
-		store.MigrateData()
+		store.MigrateData(true)
 
-		options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
+		options := store.setupOptions(getBackupRestoreOptions(store))
 
 		if !isFileExist(options.BackupPath) {
 			t.Errorf("Backup file should exist; file=%s", options.BackupPath)
@@ -101,14 +95,14 @@ func TestMigrateData(t *testing.T) {
 	})
 
 	t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false)
+		store, teardown := MustNewTestStore(false)
 		defer teardown()
 
 		store.VersionService.StoreIsUpdating(true)
 
-		store.MigrateData()
+		store.MigrateData(true)
 
-		options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
+		options := store.setupOptions(getBackupRestoreOptions(store))
 
 		if isFileExist(options.BackupPath) {
 			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
@@ -116,12 +110,12 @@ func TestMigrateData(t *testing.T) {
 	})
 
 	t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false)
+		store, teardown := MustNewTestStore(false)
 		defer teardown()
 
-		store.MigrateData()
+		store.MigrateData(true)
 
-		options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir()))
+		options := store.setupOptions(getBackupRestoreOptions(store))
 
 		if isFileExist(options.BackupPath) {
 			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
@@ -131,10 +125,10 @@ func TestMigrateData(t *testing.T) {
 }
 
 func Test_getBackupRestoreOptions(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false)
+	store, teardown := MustNewTestStore(false)
 	defer teardown()
 
-	options := getBackupRestoreOptions(store.commonBackupDir())
+	options := getBackupRestoreOptions(store)
 
 	wantDir := store.commonBackupDir()
 	if !strings.HasSuffix(options.BackupDir, wantDir) {
@@ -150,11 +144,11 @@ func Test_getBackupRestoreOptions(t *testing.T) {
 func TestRollback(t *testing.T) {
 	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
 		version := 21
-		_, store, teardown := MustNewTestStore(false)
+		store, teardown := MustNewTestStore(false)
 		defer teardown()
 		store.VersionService.StoreDBVersion(version)
 
-		_, err := store.backupWithOptions(getBackupRestoreOptions(store.commonBackupDir()))
+		_, err := store.BackupWithOptions(getBackupRestoreOptions(store))
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -176,12 +170,3 @@ func TestRollback(t *testing.T) {
 		testVersion(store, version, t)
 	})
 }
-
-// isFileExist is helper function to check for file existence
-func isFileExist(path string) bool {
-	matches, err := filepath.Glob(path)
-	if err != nil {
-		return false
-	}
-	return len(matches) > 0
-}

api/bolt/migrator/migrate_ce.go

@@ -19,8 +19,157 @@ func (m *Migrator) Migrate() error {
 		return migrationError(err, "StoreIsUpdating")
 	}
 
+	// Portainer < 1.12
+	if m.currentDBVersion < 1 {
+		err := m.updateAdminUserToDBVersion1()
+		if err != nil {
+			return migrationError(err, "updateAdminUserToDBVersion1")
+		}
+	}
+
+	// Portainer 1.12.x
+	if m.currentDBVersion < 2 {
+		err := m.updateResourceControlsToDBVersion2()
+		if err != nil {
+			return migrationError(err, "updateResourceControlsToDBVersion2")
+		}
+		err = m.updateEndpointsToDBVersion2()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToDBVersion2")
+		}
+	}
+
+	// Portainer 1.13.x
+	if m.currentDBVersion < 3 {
+		err := m.updateSettingsToDBVersion3()
+		if err != nil {
+			return migrationError(err, "updateSettingsToDBVersion3")
+		}
+	}
+
+	// Portainer 1.14.0
+	if m.currentDBVersion < 4 {
+		err := m.updateEndpointsToDBVersion4()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToDBVersion4")
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1235
+	if m.currentDBVersion < 5 {
+		err := m.updateSettingsToVersion5()
+		if err != nil {
+			return migrationError(err, "updateSettingsToVersion5")
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1236
+	if m.currentDBVersion < 6 {
+		err := m.updateSettingsToVersion6()
+		if err != nil {
+			return migrationError(err, "updateSettingsToVersion6")
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1449
+	if m.currentDBVersion < 7 {
+		err := m.updateSettingsToVersion7()
+		if err != nil {
+			return migrationError(err, "updateSettingsToVersion7")
+		}
+	}
+
+	if m.currentDBVersion < 8 {
+		err := m.updateEndpointsToVersion8()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToVersion8")
+		}
+	}
+
+	// https: //github.com/portainer/portainer/issues/1396
+	if m.currentDBVersion < 9 {
+		err := m.updateEndpointsToVersion9()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToVersion9")
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/461
+	if m.currentDBVersion < 10 {
+		err := m.updateEndpointsToVersion10()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToVersion10")
+		}
+	}
+
+	// https://github.com/portainer/portainer/issues/1906
+	if m.currentDBVersion < 11 {
+		err := m.updateEndpointsToVersion11()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToVersion11")
+		}
+	}
+
+	// Portainer 1.18.0
+	if m.currentDBVersion < 12 {
+		err := m.updateEndpointsToVersion12()
+		if err != nil {
+			return migrationError(err, "updateEndpointsToVersion12")
+		}
+
+		err = m.updateEndpointGroupsToVersion12()
+		if err != nil {
+			return migrationError(err, "updateEndpointGroupsToVersion12")
+		}
+
+		err = m.updateStacksToVersion12()
+		if err != nil {
+			return migrationError(err, "updateStacksToVersion12")
+		}
+	}
+
+	// Portainer 1.19.0
+	if m.currentDBVersion < 13 {
+		err := m.updateSettingsToVersion13()
+		if err != nil {
+			return migrationError(err, "updateSettingsToVersion13")
+		}
+	}
+
+	// Portainer 1.19.2
+	if m.currentDBVersion < 14 {
+		err := m.updateResourceControlsToDBVersion14()
+		if err != nil {
+			return migrationError(err, "updateResourceControlsToDBVersion14")
+		}
+	}
+
+	// Portainer 1.20.0
+	if m.currentDBVersion < 15 {
+		err := m.updateSettingsToDBVersion15()
+		if err != nil {
+			return migrationError(err, "updateSettingsToDBVersion15")
+		}
+
+		err = m.updateTemplatesToVersion15()
+		if err != nil {
+			return migrationError(err, "updateTemplatesToVersion15")
+		}
+	}
+
+	if m.currentDBVersion < 16 {
+		err := m.updateSettingsToDBVersion16()
+		if err != nil {
+			return migrationError(err, "updateSettingsToDBVersion16")
+		}
+	}
+
+	// Portainer 1.20.1
 	if m.currentDBVersion < 17 {
-		return migrationError(err, "migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support.")
+		err := m.updateExtensionsToDBVersion17()
+		if err != nil {
+			return migrationError(err, "updateExtensionsToDBVersion17")
+		}
 	}
 
 	// Portainer 1.21.0
@@ -88,7 +237,6 @@ func (m *Migrator) Migrate() error {
 
 	// Portainer 1.24.0
 	if m.currentDBVersion < 23 {
-		migrateLog.Info("Migrating to DB 23")
 		err := m.updateTagsToDBVersion23()
 		if err != nil {
 			return migrationError(err, "updateTagsToDBVersion23")
@@ -102,7 +250,6 @@ func (m *Migrator) Migrate() error {
 
 	// Portainer 1.24.1
 	if m.currentDBVersion < 24 {
-		migrateLog.Info("Migrating to DB 24")
 		err := m.updateSettingsToDB24()
 		if err != nil {
 			return migrationError(err, "updateSettingsToDB24")
@@ -111,7 +258,6 @@ func (m *Migrator) Migrate() error {
 
 	// Portainer 2.0.0
 	if m.currentDBVersion < 25 {
-		migrateLog.Info("Migrating to DB 25")
 		err := m.updateSettingsToDB25()
 		if err != nil {
 			return migrationError(err, "updateSettingsToDB25")
@@ -125,7 +271,6 @@ func (m *Migrator) Migrate() error {
 
 	// Portainer 2.1.0
 	if m.currentDBVersion < 26 {
-		migrateLog.Info("Migrating to DB 26")
 		err := m.updateEndpointSettingsToDB25()
 		if err != nil {
 			return migrationError(err, "updateEndpointSettingsToDB25")
@@ -134,7 +279,6 @@ func (m *Migrator) Migrate() error {
 
 	// Portainer 2.2.0
 	if m.currentDBVersion < 27 {
-		migrateLog.Info("Migrating to DB 27")
 		err := m.updateStackResourceControlToDB27()
 		if err != nil {
 			return migrationError(err, "updateStackResourceControlToDB27")
@@ -143,7 +287,6 @@ func (m *Migrator) Migrate() error {
 
 	// Portainer 2.6.0
 	if m.currentDBVersion < 30 {
-		migrateLog.Info("Migrating to DB 30")
 		err := m.migrateDBVersionToDB30()
 		if err != nil {
 			return migrationError(err, "migrateDBVersionToDB30")
@@ -160,7 +303,6 @@ func (m *Migrator) Migrate() error {
 
 	// Portainer 2.9.1, 2.9.2
 	if m.currentDBVersion < 33 {
-		migrateLog.Info("Migrating to DB 33")
 		err := m.migrateDBVersionToDB33()
 		if err != nil {
 			return migrationError(err, "migrateDBVersionToDB33")
@@ -169,7 +311,6 @@ func (m *Migrator) Migrate() error {
 
 	// Portainer 2.10
 	if m.currentDBVersion < 34 {
-		migrateLog.Info("Migrating to DB 34")
 		if err := m.migrateDBVersionToDB34(); err != nil {
 			return migrationError(err, "migrateDBVersionToDB34")
 		}
@@ -177,7 +318,6 @@ func (m *Migrator) Migrate() error {
 
 	// Portainer 2.9.3 (yep out of order, but 2.10 is EE only)
 	if m.currentDBVersion < 35 {
-		migrateLog.Info("Migrating to DB 35")
 		if err := m.migrateDBVersionToDB35(); err != nil {
 			return migrationError(err, "migrateDBVersionToDB35")
 		}

api/bolt/migrator/migrate_dbversion0.go

@@ -0,0 +1,37 @@
+package migrator
+
+import (
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/bolt/user"
+)
+
+func (m *Migrator) updateAdminUserToDBVersion1() error {
+	u, err := m.userService.UserByUsername("admin")
+	if err == nil {
+		admin := &portainer.User{
+			Username: "admin",
+			Password: u.Password,
+			Role:     portainer.AdministratorRole,
+		}
+		err = m.userService.CreateUser(admin)
+		if err != nil {
+			return err
+		}
+		err = m.removeLegacyAdminUser()
+		if err != nil {
+			return err
+		}
+	} else if err != nil && err != errors.ErrObjectNotFound {
+		return err
+	}
+	return nil
+}
+
+func (m *Migrator) removeLegacyAdminUser() error {
+	return m.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(user.BucketName))
+		return bucket.Delete([]byte("admin"))
+	})
+}

api/bolt/migrator/migrate_dbversion1.go

@@ -0,0 +1,103 @@
+package migrator
+
+import (
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+)
+
+func (m *Migrator) updateResourceControlsToDBVersion2() error {
+	legacyResourceControls, err := m.retrieveLegacyResourceControls()
+	if err != nil {
+		return err
+	}
+
+	for _, resourceControl := range legacyResourceControls {
+		resourceControl.SubResourceIDs = []string{}
+		resourceControl.TeamAccesses = []portainer.TeamResourceAccess{}
+
+		owner, err := m.userService.User(resourceControl.OwnerID)
+		if err != nil {
+			return err
+		}
+
+		if owner.Role == portainer.AdministratorRole {
+			resourceControl.AdministratorsOnly = true
+			resourceControl.UserAccesses = []portainer.UserResourceAccess{}
+		} else {
+			resourceControl.AdministratorsOnly = false
+			userAccess := portainer.UserResourceAccess{
+				UserID:      resourceControl.OwnerID,
+				AccessLevel: portainer.ReadWriteAccessLevel,
+			}
+			resourceControl.UserAccesses = []portainer.UserResourceAccess{userAccess}
+		}
+
+		err = m.resourceControlService.CreateResourceControl(&resourceControl)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateEndpointsToDBVersion2() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.AuthorizedTeams = []portainer.TeamID{}
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) retrieveLegacyResourceControls() ([]portainer.ResourceControl, error) {
+	legacyResourceControls := make([]portainer.ResourceControl, 0)
+	err := m.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte("containerResourceControl"))
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var resourceControl portainer.ResourceControl
+			err := internal.UnmarshalObject(v, &resourceControl)
+			if err != nil {
+				return err
+			}
+			resourceControl.Type = portainer.ContainerResourceControl
+			legacyResourceControls = append(legacyResourceControls, resourceControl)
+		}
+
+		bucket = tx.Bucket([]byte("serviceResourceControl"))
+		cursor = bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var resourceControl portainer.ResourceControl
+			err := internal.UnmarshalObject(v, &resourceControl)
+			if err != nil {
+				return err
+			}
+			resourceControl.Type = portainer.ServiceResourceControl
+			legacyResourceControls = append(legacyResourceControls, resourceControl)
+		}
+
+		bucket = tx.Bucket([]byte("volumeResourceControl"))
+		cursor = bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var resourceControl portainer.ResourceControl
+			err := internal.UnmarshalObject(v, &resourceControl)
+			if err != nil {
+				return err
+			}
+			resourceControl.Type = portainer.VolumeResourceControl
+			legacyResourceControls = append(legacyResourceControls, resourceControl)
+		}
+		return nil
+	})
+	return legacyResourceControls, err
+}

api/bolt/migrator/migrate_dbversion10.go

@@ -0,0 +1,28 @@
+package migrator
+
+import "github.com/portainer/portainer/api"
+
+func (m *Migrator) updateEndpointsToVersion11() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		if endpoint.Type == portainer.AgentOnDockerEnvironment {
+			endpoint.TLSConfig.TLS = true
+			endpoint.TLSConfig.TLSSkipVerify = true
+		} else {
+			if endpoint.TLSConfig.TLSSkipVerify && !endpoint.TLSConfig.TLS {
+				endpoint.TLSConfig.TLSSkipVerify = false
+			}
+		}
+
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion11.go

@@ -0,0 +1,127 @@
+package migrator
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/stack"
+)
+
+func (m *Migrator) updateEndpointsToVersion12() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.Tags = []string{}
+
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateEndpointGroupsToVersion12() error {
+	legacyEndpointGroups, err := m.endpointGroupService.EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	for _, group := range legacyEndpointGroups {
+		group.Tags = []string{}
+
+		err = m.endpointGroupService.UpdateEndpointGroup(group.ID, &group)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type legacyStack struct {
+	ID          string               `json:"Id"`
+	Name        string               `json:"Name"`
+	EndpointID  portainer.EndpointID `json:"EndpointId"`
+	SwarmID     string               `json:"SwarmId"`
+	EntryPoint  string               `json:"EntryPoint"`
+	Env         []portainer.Pair     `json:"Env"`
+	ProjectPath string
+}
+
+func (m *Migrator) updateStacksToVersion12() error {
+	legacyStacks, err := m.retrieveLegacyStacks()
+	if err != nil {
+		return err
+	}
+
+	for _, legacyStack := range legacyStacks {
+		err := m.convertLegacyStack(&legacyStack)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (m *Migrator) convertLegacyStack(s *legacyStack) error {
+	stackID := m.stackService.GetNextIdentifier()
+	stack := &portainer.Stack{
+		ID:         portainer.StackID(stackID),
+		Name:       s.Name,
+		Type:       portainer.DockerSwarmStack,
+		SwarmID:    s.SwarmID,
+		EndpointID: 0,
+		EntryPoint: s.EntryPoint,
+		Env:        s.Env,
+	}
+
+	stack.ProjectPath = strings.Replace(s.ProjectPath, s.ID, strconv.Itoa(stackID), 1)
+	err := m.fileService.Rename(s.ProjectPath, stack.ProjectPath)
+	if err != nil {
+		return err
+	}
+
+	err = m.deleteLegacyStack(s.ID)
+	if err != nil {
+		return err
+	}
+
+	return m.stackService.CreateStack(stack)
+}
+
+func (m *Migrator) deleteLegacyStack(legacyID string) error {
+	return m.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(stack.BucketName))
+		return bucket.Delete([]byte(legacyID))
+	})
+}
+
+func (m *Migrator) retrieveLegacyStacks() ([]legacyStack, error) {
+	var legacyStacks = make([]legacyStack, 0)
+	err := m.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(stack.BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var stack legacyStack
+			err := internal.UnmarshalObject(v, &stack)
+			if err != nil {
+				return err
+			}
+			legacyStacks = append(legacyStacks, stack)
+		}
+
+		return nil
+	})
+
+	return legacyStacks, err
+}

api/bolt/migrator/migrate_dbversion12.go

@@ -0,0 +1,17 @@
+package migrator
+
+import "github.com/portainer/portainer/api"
+
+func (m *Migrator) updateSettingsToVersion13() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.LDAPSettings.AutoCreateUsers = false
+	legacySettings.LDAPSettings.GroupSearchSettings = []portainer.LDAPGroupSearchSettings{
+		portainer.LDAPGroupSearchSettings{},
+	}
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion13.go

@@ -0,0 +1,19 @@
+package migrator
+
+func (m *Migrator) updateResourceControlsToDBVersion14() error {
+	resourceControls, err := m.resourceControlService.ResourceControls()
+	if err != nil {
+		return err
+	}
+
+	for _, resourceControl := range resourceControls {
+		if resourceControl.AdministratorsOnly == true {
+			err = m.resourceControlService.DeleteResourceControl(resourceControl.ID)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion14.go

@@ -0,0 +1,16 @@
+package migrator
+
+func (m *Migrator) updateSettingsToDBVersion15() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.EnableHostManagementFeatures = false
+	return m.settingsService.UpdateSettings(legacySettings)
+}
+
+func (m *Migrator) updateTemplatesToVersion15() error {
+	// Removed with the entire template management layer, part of https://github.com/portainer/portainer/issues/3707
+	return nil
+}

api/bolt/migrator/migrate_dbversion15.go

@@ -0,0 +1,14 @@
+package migrator
+
+func (m *Migrator) updateSettingsToDBVersion16() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	if legacySettings.SnapshotInterval == "" {
+		legacySettings.SnapshotInterval = "5m"
+	}
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion16.go

@@ -0,0 +1,19 @@
+package migrator
+
+func (m *Migrator) updateExtensionsToDBVersion17() error {
+	legacyExtensions, err := m.extensionService.Extensions()
+	if err != nil {
+		return err
+	}
+
+	for _, extension := range legacyExtensions {
+		extension.License.Valid = true
+
+		err = m.extensionService.Persist(&extension)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion2.go

@@ -0,0 +1,20 @@
+package migrator
+
+import "github.com/portainer/portainer/api"
+
+func (m *Migrator) updateSettingsToDBVersion3() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.AuthenticationMethod = portainer.AuthenticationInternal
+	legacySettings.LDAPSettings = portainer.LDAPSettings{
+		TLSConfig: portainer.TLSConfiguration{},
+		SearchSettings: []portainer.LDAPSearchSettings{
+			portainer.LDAPSearchSettings{},
+		},
+	}
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion22.go

@@ -1,9 +1,8 @@
 package migrator
 
-import portainer "github.com/portainer/portainer/api"
+import "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateTagsToDBVersion23() error {
-	migrateLog.Info("Updating tags")
 	tags, err := m.tagService.Tags()
 	if err != nil {
 		return err
@@ -21,7 +20,6 @@ func (m *Migrator) updateTagsToDBVersion23() error {
 }
 
 func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
-	migrateLog.Info("Updating endpoints and endpoint groups")
 	tags, err := m.tagService.Tags()
 	if err != nil {
 		return err
@@ -57,7 +55,7 @@ func (m *Migrator) updateEndpointsAndEndpointGroupsToDBVersion23() error {
 			EdgeStacks: map[portainer.EdgeStackID]bool{},
 		}
 
-		err = m.endpointRelationService.Create(relation)
+		err = m.endpointRelationService.CreateEndpointRelation(relation)
 		if err != nil {
 			return err
 		}

api/bolt/migrator/migrate_dbversion23.go

@@ -3,8 +3,6 @@ package migrator
 import portainer "github.com/portainer/portainer/api"
 
 func (m *Migrator) updateSettingsToDB24() error {
-	migrateLog.Info("Updating Settings")
-
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {
 		return err
@@ -18,7 +16,6 @@ func (m *Migrator) updateSettingsToDB24() error {
 }
 
 func (m *Migrator) updateStacksToDB24() error {
-	migrateLog.Info("Updating stacks")
 	stacks, err := m.stackService.Stacks()
 	if err != nil {
 		return err

api/bolt/migrator/migrate_dbversion24.go

@@ -1,12 +1,10 @@
 package migrator
 
 import (
-	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api"
 )
 
 func (m *Migrator) updateSettingsToDB25() error {
-	migrateLog.Info("Updating settings")
-
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {
 		return err

api/bolt/migrator/migrate_dbversion25.go

@@ -5,7 +5,6 @@ import (
 )
 
 func (m *Migrator) updateEndpointSettingsToDB25() error {
-	migrateLog.Info("Updating endpoint settings")
 	settings, err := m.settingsService.Settings()
 	if err != nil {
 		return err

api/bolt/migrator/migrate_dbversion26.go

@@ -2,12 +2,11 @@ package migrator
 
 import (
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/internal/stackutils"
 )
 
 func (m *Migrator) updateStackResourceControlToDB27() error {
-	migrateLog.Info("Updating stack resource controls")
 	resourceControls, err := m.resourceControlService.ResourceControls()
 	if err != nil {
 		return err

api/bolt/migrator/migrate_dbversion29.go

@@ -1,23 +1,18 @@
 package migrator
 
 func (m *Migrator) migrateDBVersionToDB30() error {
-	migrateLog.Info("Updating legacy settings")
-	if err := m.MigrateSettingsToDB30(); err != nil {
+	if err := m.migrateSettingsToDB30(); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-// so setting to false and "", is what would happen without this code
-// I'm going to bet there's zero point to changing the value inthe DB
-// Public for testing
-func (m *Migrator) MigrateSettingsToDB30() error {
+func (m *Migrator) migrateSettingsToDB30() error {
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {
 		return err
 	}
-
 	legacySettings.OAuthSettings.SSO = false
 	legacySettings.OAuthSettings.LogoutURI = ""
 	return m.settingsService.UpdateSettings(legacySettings)

api/bolt/migrator/migrate_dbversion29_test.go

@@ -0,0 +1,95 @@
+package migrator
+
+import (
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/boltdb/bolt"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/settings"
+)
+
+var (
+	testingDBStorePath string
+	testingDBFileName  string
+	dummyLogoURL       string
+	dbConn             *bolt.DB
+	settingsService    *settings.Service
+)
+
+// initTestingDBConn creates a raw bolt DB connection
+// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
+func initTestingDBConn(storePath, fileName string) (*bolt.DB, error) {
+	databasePath := path.Join(storePath, fileName)
+	dbConn, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return nil, err
+	}
+	return dbConn, nil
+}
+
+// initTestingDBConn creates a settings service with raw bolt DB connection
+// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
+func initTestingSettingsService(dbConn *bolt.DB, preSetObj map[string]interface{}) (*settings.Service, error) {
+	internalDBConn := &internal.DbConnection{
+		DB: dbConn,
+	}
+	settingsService, err := settings.NewService(internalDBConn)
+	if err != nil {
+		return nil, err
+	}
+	//insert a obj
+	if err := internal.UpdateObject(internalDBConn, "settings", []byte("SETTINGS"), preSetObj); err != nil {
+		return nil, err
+	}
+	return settingsService, nil
+}
+
+func setup() error {
+	testingDBStorePath, _ = os.Getwd()
+	testingDBFileName = "portainer-ee-mig-30.db"
+	dummyLogoURL = "example.com"
+	var err error
+	dbConn, err = initTestingDBConn(testingDBStorePath, testingDBFileName)
+	if err != nil {
+		return err
+	}
+	dummySettingsObj := map[string]interface{}{
+		"LogoURL": dummyLogoURL,
+	}
+	settingsService, err = initTestingSettingsService(dbConn, dummySettingsObj)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func TestMigrateSettings(t *testing.T) {
+	if err := setup(); err != nil {
+		t.Errorf("failed to complete testing setups, err: %v", err)
+	}
+	defer dbConn.Close()
+	defer os.Remove(testingDBFileName)
+	m := &Migrator{
+		db:              dbConn,
+		settingsService: settingsService,
+	}
+	if err := m.migrateSettingsToDB30(); err != nil {
+		t.Errorf("failed to update settings: %v", err)
+	}
+	updatedSettings, err := m.settingsService.Settings()
+	if err != nil {
+		t.Errorf("failed to retrieve the updated settings: %v", err)
+	}
+	if updatedSettings.LogoURL != dummyLogoURL {
+		t.Errorf("unexpected value changes in the updated settings, want LogoURL value: %s, got LogoURL value: %s", dummyLogoURL, updatedSettings.LogoURL)
+	}
+	if updatedSettings.OAuthSettings.SSO != false {
+		t.Errorf("unexpected default OAuth SSO setting, want: false, got: %t", updatedSettings.OAuthSettings.SSO)
+	}
+	if updatedSettings.OAuthSettings.LogoutURI != "" {
+		t.Errorf("unexpected default OAuth HideInternalAuth setting, want:, got: %s", updatedSettings.OAuthSettings.LogoutURI)
+	}
+}

api/bolt/migrator/migrate_dbversion3.go

@@ -0,0 +1,28 @@
+package migrator
+
+import "github.com/portainer/portainer/api"
+
+func (m *Migrator) updateEndpointsToDBVersion4() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.TLSConfig = portainer.TLSConfiguration{}
+		if endpoint.TLS {
+			endpoint.TLSConfig.TLS = true
+			endpoint.TLSConfig.TLSSkipVerify = false
+			endpoint.TLSConfig.TLSCACertPath = endpoint.TLSCACertPath
+			endpoint.TLSConfig.TLSCertPath = endpoint.TLSCertPath
+			endpoint.TLSConfig.TLSKeyPath = endpoint.TLSKeyPath
+		}
+
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion31.go

@@ -2,38 +2,33 @@ package migrator
 
 import (
 	"fmt"
-	"github.com/portainer/portainer/api/dataservices/errors"
 	"log"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/internal/endpointutils"
 	snapshotutils "github.com/portainer/portainer/api/internal/snapshot"
 )
 
 func (m *Migrator) migrateDBVersionToDB32() error {
-	migrateLog.Info("Updating registries")
 	err := m.updateRegistriesToDB32()
 	if err != nil {
 		return err
 	}
 
-	migrateLog.Info("Updating dockerhub")
 	err = m.updateDockerhubToDB32()
 	if err != nil {
 		return err
 	}
 
-	migrateLog.Info("Updating resource controls")
 	if err := m.updateVolumeResourceControlToDB32(); err != nil {
 		return err
 	}
 
-	migrateLog.Info("Updating kubeconfig expiry")
 	if err := m.kubeconfigExpiryToDB32(); err != nil {
 		return err
 	}
 
-	migrateLog.Info("Setting default helm repository url")
 	if err := m.helmRepositoryURLToDB32(); err != nil {
 		return err
 	}
@@ -168,7 +163,7 @@ func (m *Migrator) updateDockerhubToDB32() error {
 		}
 	}
 
-	return m.registryService.Create(registry)
+	return m.registryService.CreateRegistry(registry)
 }
 
 func (m *Migrator) updateVolumeResourceControlToDB32() error {
@@ -207,7 +202,7 @@ func (m *Migrator) updateVolumeResourceControlToDB32() error {
 
 		endpointDockerID, err := snapshotutils.FetchDockerID(snapshot)
 		if err != nil {
-			log.Printf("[WARN] [database,migrator,v31] [message: failed fetching environment docker id] [err: %s]", err)
+			log.Printf("[WARN] [bolt,migrator,v31] [message: failed fetching environment docker id] [err: %s]", err)
 			continue
 		}
 

api/bolt/migrator/migrate_dbversion32.go

@@ -16,7 +16,6 @@ func (m *Migrator) migrateSettingsToDB33() error {
 		return err
 	}
 
-	migrateLog.Info("Setting default kubectl shell image")
 	settings.KubectlShellImage = portainer.DefaultKubectlShellImage
 	return m.settingsService.UpdateSettings(settings)
 }

api/bolt/migrator/migrate_dbversion33.go

@@ -1,10 +1,11 @@
 package migrator
 
-import "github.com/portainer/portainer/api/dataservices"
+import (
+	portainer "github.com/portainer/portainer/api"
+)
 
 func (m *Migrator) migrateDBVersionToDB34() error {
-	migrateLog.Info("Migrating stacks")
-	err := MigrateStackEntryPoint(m.stackService)
+	err := migrateStackEntryPoint(m.stackService)
 	if err != nil {
 		return err
 	}
@@ -12,8 +13,7 @@ func (m *Migrator) migrateDBVersionToDB34() error {
 	return nil
 }
 
-// MigrateStackEntryPoint exported for testing (blah.)
-func MigrateStackEntryPoint(stackService dataservices.StackService) error {
+func migrateStackEntryPoint(stackService portainer.StackService) error {
 	stacks, err := stackService.Stacks()
 	if err != nil {
 		return err

api/bolt/migrator/migrate_dbversion33_test.go

@@ -1,19 +1,25 @@
-package datastore
+package migrator
 
 import (
+	"path"
 	"testing"
+	"time"
 
+	"github.com/boltdb/bolt"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/datastore/migrator"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/stack"
 	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestMigrateStackEntryPoint(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false)
-	defer teardown()
+	dbConn, err := bolt.Open(path.Join(t.TempDir(), "portainer-ee-mig-34.db"), 0600, &bolt.Options{Timeout: 1 * time.Second})
+	assert.NoError(t, err, "failed to init testing DB connection")
+	defer dbConn.Close()
 
-	stackService := store.Stack()
+	stackService, err := stack.NewService(&internal.DbConnection{DB: dbConn})
+	assert.NoError(t, err, "failed to init testing Stack service")
 
 	stacks := []*portainer.Stack{
 		{
@@ -28,25 +34,17 @@ func TestMigrateStackEntryPoint(t *testing.T) {
 	}
 
 	for _, s := range stacks {
-		err := stackService.Create(s)
+		err := stackService.CreateStack(s)
 		assert.NoError(t, err, "failed to create stack")
 	}
 
+	err = migrateStackEntryPoint(stackService)
+	assert.NoError(t, err, "failed to migrate entry point to Git ConfigFilePath")
+
 	s, err := stackService.Stack(1)
 	assert.NoError(t, err)
 	assert.Nil(t, s.GitConfig, "first stack should not have git config")
 
-	s, err = stackService.Stack(2)
-	assert.NoError(t, err)
-	assert.Equal(t, "", s.GitConfig.ConfigFilePath, "not migrated yet migrated")
-
-	err = migrator.MigrateStackEntryPoint(stackService)
-	assert.NoError(t, err, "failed to migrate entry point to Git ConfigFilePath")
-
-	s, err = stackService.Stack(1)
-	assert.NoError(t, err)
-	assert.Nil(t, s.GitConfig, "first stack should not have git config")
-
 	s, err = stackService.Stack(2)
 	assert.NoError(t, err)
 	assert.Equal(t, "dir/sub/compose.yml", s.GitConfig.ConfigFilePath, "second stack should have config file path migrated")

api/bolt/migrator/migrate_dbversion34.go

@@ -3,7 +3,6 @@ package migrator
 func (m *Migrator) migrateDBVersionToDB35() error {
 	// These should have been migrated already, but due to an earlier bug and a bunch of duplicates,
 	// calling it again will now fix the issue as the function has been repaired.
-	migrateLog.Info("Updating dockerhub registries")
 	err := m.updateDockerhubToDB32()
 	if err != nil {
 		return err

api/bolt/migrator/migrate_dbversion34_test.go

@@ -0,0 +1,108 @@
+package migrator
+
+import (
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/boltdb/bolt"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/dockerhub"
+	"github.com/portainer/portainer/api/bolt/endpoint"
+	"github.com/portainer/portainer/api/bolt/internal"
+	"github.com/portainer/portainer/api/bolt/registry"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	db35TestFile = "portainer-mig-35.db"
+	username     = "portainer"
+	password     = "password"
+)
+
+func setupDB35Test(t *testing.T) *Migrator {
+	is := assert.New(t)
+	dbConn, err := bolt.Open(path.Join(t.TempDir(), db35TestFile), 0600, &bolt.Options{Timeout: 1 * time.Second})
+	is.NoError(err, "failed to init testing DB connection")
+
+	// Create an old style dockerhub authenticated account
+	dockerhubService, err := dockerhub.NewService(&internal.DbConnection{DB: dbConn})
+	is.NoError(err, "failed to init testing registry service")
+	err = dockerhubService.UpdateDockerHub(&portainer.DockerHub{true, username, password})
+	is.NoError(err, "failed to create dockerhub account")
+
+	registryService, err := registry.NewService(&internal.DbConnection{DB: dbConn})
+	is.NoError(err, "failed to init testing registry service")
+
+	endpointService, err := endpoint.NewService(&internal.DbConnection{DB: dbConn})
+	is.NoError(err, "failed to init endpoint service")
+
+	m := &Migrator{
+		db:               dbConn,
+		dockerhubService: dockerhubService,
+		registryService:  registryService,
+		endpointService:  endpointService,
+	}
+
+	return m
+}
+
+// TestUpdateDockerhubToDB32 tests a normal upgrade
+func TestUpdateDockerhubToDB32(t *testing.T) {
+	is := assert.New(t)
+	m := setupDB35Test(t)
+	defer m.db.Close()
+	defer os.Remove(db35TestFile)
+
+	if err := m.updateDockerhubToDB32(); err != nil {
+		t.Errorf("failed to update settings: %v", err)
+	}
+
+	// Verify we have a single registry were created
+	registries, err := m.registryService.Registries()
+	is.NoError(err, "failed to read registries from the RegistryService")
+	is.Equal(len(registries), 1, "only one migrated registry expected")
+}
+
+// TestUpdateDockerhubToDB32_with_duplicate_migrations tests an upgrade where in earlier versions a broken migration
+// created a large number of duplicate "dockerhub migrated" registry entries.
+func TestUpdateDockerhubToDB32_with_duplicate_migrations(t *testing.T) {
+	is := assert.New(t)
+	m := setupDB35Test(t)
+	defer m.db.Close()
+	defer os.Remove(db35TestFile)
+
+	// Create lots of duplicate entries...
+	registry := &portainer.Registry{
+		Type:             portainer.DockerHubRegistry,
+		Name:             "Dockerhub (authenticated - migrated)",
+		URL:              "docker.io",
+		Authentication:   true,
+		Username:         "portainer",
+		Password:         "password",
+		RegistryAccesses: portainer.RegistryAccesses{},
+	}
+
+	for i := 1; i < 150; i++ {
+		err := m.registryService.CreateRegistry(registry)
+		assert.NoError(t, err, "create registry failed")
+	}
+
+	// Verify they were created
+	registries, err := m.registryService.Registries()
+	is.NoError(err, "failed to read registries from the RegistryService")
+	is.Condition(func() bool {
+		return len(registries) > 1
+	}, "expected multiple duplicate registry entries")
+
+	// Now run the migrator
+	if err := m.updateDockerhubToDB32(); err != nil {
+		t.Errorf("failed to update settings: %v", err)
+	}
+
+	// Verify we have a single registry were created
+	registries, err = m.registryService.Registries()
+	is.NoError(err, "failed to read registries from the RegistryService")
+	is.Equal(len(registries), 1, "only one migrated registry expected")
+}

api/bolt/migrator/migrate_dbversion4.go

@@ -0,0 +1,11 @@
+package migrator
+
+func (m *Migrator) updateSettingsToVersion5() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.AllowBindMountsForRegularUsers = true
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion5.go

@@ -0,0 +1,11 @@
+package migrator
+
+func (m *Migrator) updateSettingsToVersion6() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.AllowPrivilegedModeForRegularUsers = true
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion6.go

@@ -0,0 +1,11 @@
+package migrator
+
+func (m *Migrator) updateSettingsToVersion7() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+	legacySettings.DisplayDonationHeader = true
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrate_dbversion7.go

@@ -0,0 +1,20 @@
+package migrator
+
+import "github.com/portainer/portainer/api"
+
+func (m *Migrator) updateEndpointsToVersion8() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.Extensions = []portainer.EndpointExtension{}
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion8.go

@@ -0,0 +1,20 @@
+package migrator
+
+import "github.com/portainer/portainer/api"
+
+func (m *Migrator) updateEndpointsToVersion9() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.GroupID = portainer.EndpointGroupID(1)
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrate_dbversion9.go

@@ -0,0 +1,20 @@
+package migrator
+
+import "github.com/portainer/portainer/api"
+
+func (m *Migrator) updateEndpointsToVersion10() error {
+	legacyEndpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.Type = portainer.DockerEnvironment
+		err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator/migrator.go

@@ -1,38 +1,39 @@
 package migrator
 
 import (
+	"github.com/boltdb/bolt"
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/dockerhub"
-	"github.com/portainer/portainer/api/dataservices/endpoint"
-	"github.com/portainer/portainer/api/dataservices/endpointgroup"
-	"github.com/portainer/portainer/api/dataservices/endpointrelation"
-	"github.com/portainer/portainer/api/dataservices/extension"
-	"github.com/portainer/portainer/api/dataservices/fdoprofile"
-	"github.com/portainer/portainer/api/dataservices/registry"
-	"github.com/portainer/portainer/api/dataservices/resourcecontrol"
-	"github.com/portainer/portainer/api/dataservices/role"
-	"github.com/portainer/portainer/api/dataservices/schedule"
-	"github.com/portainer/portainer/api/dataservices/settings"
-	"github.com/portainer/portainer/api/dataservices/stack"
-	"github.com/portainer/portainer/api/dataservices/tag"
-	"github.com/portainer/portainer/api/dataservices/teammembership"
-	"github.com/portainer/portainer/api/dataservices/user"
-	"github.com/portainer/portainer/api/dataservices/version"
-	plog "github.com/portainer/portainer/api/datastore/log"
+	"github.com/portainer/portainer/api/bolt/dockerhub"
+	"github.com/portainer/portainer/api/bolt/endpoint"
+	"github.com/portainer/portainer/api/bolt/endpointgroup"
+	"github.com/portainer/portainer/api/bolt/endpointrelation"
+	"github.com/portainer/portainer/api/bolt/extension"
+	plog "github.com/portainer/portainer/api/bolt/log"
+	"github.com/portainer/portainer/api/bolt/registry"
+	"github.com/portainer/portainer/api/bolt/resourcecontrol"
+	"github.com/portainer/portainer/api/bolt/role"
+	"github.com/portainer/portainer/api/bolt/schedule"
+	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/stack"
+	"github.com/portainer/portainer/api/bolt/tag"
+	"github.com/portainer/portainer/api/bolt/teammembership"
+	"github.com/portainer/portainer/api/bolt/user"
+	"github.com/portainer/portainer/api/bolt/version"
 	"github.com/portainer/portainer/api/internal/authorization"
 )
 
-var migrateLog = plog.NewScopedLog("database, migrate")
+var migrateLog = plog.NewScopedLog("bolt, migrate")
 
 type (
 	// Migrator defines a service to migrate data after a Portainer version update.
 	Migrator struct {
-		currentDBVersion        int
+		db               *bolt.DB
+		currentDBVersion int
+
 		endpointGroupService    *endpointgroup.Service
 		endpointService         *endpoint.Service
 		endpointRelationService *endpointrelation.Service
 		extensionService        *extension.Service
-		fdoProfilesService      *fdoprofile.Service
 		registryService         *registry.Service
 		resourceControlService  *resourcecontrol.Service
 		roleService             *role.Service
@@ -48,14 +49,14 @@ type (
 		dockerhubService        *dockerhub.Service
 	}
 
-	// MigratorParameters represents the required parameters to create a new Migrator instance.
-	MigratorParameters struct {
+	// Parameters represents the required parameters to create a new Migrator instance.
+	Parameters struct {
+		DB                      *bolt.DB
 		DatabaseVersion         int
 		EndpointGroupService    *endpointgroup.Service
 		EndpointService         *endpoint.Service
 		EndpointRelationService *endpointrelation.Service
 		ExtensionService        *extension.Service
-		FDOProfilesService      *fdoprofile.Service
 		RegistryService         *registry.Service
 		ResourceControlService  *resourcecontrol.Service
 		RoleService             *role.Service
@@ -73,14 +74,14 @@ type (
 )
 
 // NewMigrator creates a new Migrator.
-func NewMigrator(parameters *MigratorParameters) *Migrator {
+func NewMigrator(parameters *Parameters) *Migrator {
 	return &Migrator{
+		db:                      parameters.DB,
 		currentDBVersion:        parameters.DatabaseVersion,
 		endpointGroupService:    parameters.EndpointGroupService,
 		endpointService:         parameters.EndpointService,
 		endpointRelationService: parameters.EndpointRelationService,
 		extensionService:        parameters.ExtensionService,
-		fdoProfilesService:      parameters.FDOProfilesService,
 		registryService:         parameters.RegistryService,
 		resourceControlService:  parameters.ResourceControlService,
 		roleService:             parameters.RoleService,

api/bolt/registry/registry.go

@@ -0,0 +1,95 @@
+package registry
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "registries"
+)
+
+// Service represents a service for managing environment(endpoint) data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// Registry returns an registry by ID.
+func (service *Service) Registry(ID portainer.RegistryID) (*portainer.Registry, error) {
+	var registry portainer.Registry
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &registry)
+	if err != nil {
+		return nil, err
+	}
+
+	return &registry, nil
+}
+
+// Registries returns an array containing all the registries.
+func (service *Service) Registries() ([]portainer.Registry, error) {
+	var registries = make([]portainer.Registry, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var registry portainer.Registry
+			err := internal.UnmarshalObject(v, &registry)
+			if err != nil {
+				return err
+			}
+			registries = append(registries, registry)
+		}
+
+		return nil
+	})
+
+	return registries, err
+}
+
+// CreateRegistry creates a new registry.
+func (service *Service) CreateRegistry(registry *portainer.Registry) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		registry.ID = portainer.RegistryID(id)
+
+		data, err := internal.MarshalObject(registry)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(registry.ID)), data)
+	})
+}
+
+// UpdateRegistry updates an registry.
+func (service *Service) UpdateRegistry(ID portainer.RegistryID, registry *portainer.Registry) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, registry)
+}
+
+// DeleteRegistry deletes an registry.
+func (service *Service) DeleteRegistry(ID portainer.RegistryID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}

api/bolt/resourcecontrol/resourcecontrol.go

@@ -1,10 +1,10 @@
 package resourcecontrol
 
 import (
-	"fmt"
-
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
 )
 
 const (
@@ -14,16 +14,12 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection portainer.Connection) (*Service, error) {
-	err := connection.SetServiceName(BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
@@ -36,9 +32,9 @@ func NewService(connection portainer.Connection) (*Service, error) {
 // ResourceControl returns a ResourceControl object by ID
 func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portainer.ResourceControl, error) {
 	var resourceControl portainer.ResourceControl
-	identifier := service.connection.ConvertToKey(int(ID))
+	identifier := internal.Itob(int(ID))
 
-	err := service.connection.GetObject(BucketName, identifier, &resourceControl)
+	err := internal.GetObject(service.connection, BucketName, identifier, &resourceControl)
 	if err != nil {
 		return nil, err
 	}
@@ -51,76 +47,85 @@ func (service *Service) ResourceControl(ID portainer.ResourceControlID) (*portai
 // if no ResourceControl was found.
 func (service *Service) ResourceControlByResourceIDAndType(resourceID string, resourceType portainer.ResourceControlType) (*portainer.ResourceControl, error) {
 	var resourceControl *portainer.ResourceControl
-	stop := fmt.Errorf("ok")
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.ResourceControl{},
-		func(obj interface{}) (interface{}, error) {
-			rc, ok := obj.(*portainer.ResourceControl)
-			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var rc portainer.ResourceControl
+			err := internal.UnmarshalObject(v, &rc)
+			if err != nil {
+				return err
 			}
 
 			if rc.ResourceID == resourceID && rc.Type == resourceType {
-				resourceControl = rc
-				return nil, stop
+				resourceControl = &rc
+				break
 			}
 
 			for _, subResourceID := range rc.SubResourceIDs {
 				if subResourceID == resourceID {
-					resourceControl = rc
-					return nil, stop
+					resourceControl = &rc
+					break
 				}
 			}
-			return &portainer.ResourceControl{}, nil
-		})
-	if err == stop {
-		return resourceControl, nil
-	}
+		}
 
-	return nil, err
+		return nil
+	})
+
+	return resourceControl, err
 }
 
 // ResourceControls returns all the ResourceControl objects
 func (service *Service) ResourceControls() ([]portainer.ResourceControl, error) {
 	var rcs = make([]portainer.ResourceControl, 0)
 
-	err := service.connection.GetAll(
-		BucketName,
-		&portainer.ResourceControl{},
-		func(obj interface{}) (interface{}, error) {
-			rc, ok := obj.(*portainer.ResourceControl)
-			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to ResourceControl object")
-				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var resourceControl portainer.ResourceControl
+			err := internal.UnmarshalObject(v, &resourceControl)
+			if err != nil {
+				return err
 			}
-			rcs = append(rcs, *rc)
-			return &portainer.ResourceControl{}, nil
-		})
+			rcs = append(rcs, resourceControl)
+		}
+
+		return nil
+	})
 
 	return rcs, err
 }
 
 // CreateResourceControl creates a new ResourceControl object
-func (service *Service) Create(resourceControl *portainer.ResourceControl) error {
-	return service.connection.CreateObject(
-		BucketName,
-		func(id uint64) (int, interface{}) {
-			resourceControl.ID = portainer.ResourceControlID(id)
-			return int(resourceControl.ID), resourceControl
-		},
-	)
+func (service *Service) CreateResourceControl(resourceControl *portainer.ResourceControl) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		resourceControl.ID = portainer.ResourceControlID(id)
+
+		data, err := internal.MarshalObject(resourceControl)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(resourceControl.ID)), data)
+	})
 }
 
 // UpdateResourceControl saves a ResourceControl object.
 func (service *Service) UpdateResourceControl(ID portainer.ResourceControlID, resourceControl *portainer.ResourceControl) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.UpdateObject(BucketName, identifier, resourceControl)
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, resourceControl)
 }
 
 // DeleteResourceControl deletes a ResourceControl object by ID
 func (service *Service) DeleteResourceControl(ID portainer.ResourceControlID) error {
-	identifier := service.connection.ConvertToKey(int(ID))
-	return service.connection.DeleteObject(BucketName, identifier)
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
 }

api/bolt/role/role.go

@@ -0,0 +1,89 @@
+package role
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "roles"
+)
+
+// Service represents a service for managing environment(endpoint) data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// Role returns a Role by ID
+func (service *Service) Role(ID portainer.RoleID) (*portainer.Role, error) {
+	var set portainer.Role
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &set)
+	if err != nil {
+		return nil, err
+	}
+
+	return &set, nil
+}
+
+// Roles return an array containing all the sets.
+func (service *Service) Roles() ([]portainer.Role, error) {
+	var sets = make([]portainer.Role, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var set portainer.Role
+			err := internal.UnmarshalObject(v, &set)
+			if err != nil {
+				return err
+			}
+			sets = append(sets, set)
+		}
+
+		return nil
+	})
+
+	return sets, err
+}
+
+// CreateRole creates a new Role.
+func (service *Service) CreateRole(role *portainer.Role) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		role.ID = portainer.RoleID(id)
+
+		data, err := internal.MarshalObject(role)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(role.ID)), data)
+	})
+}
+
+// UpdateRole updates a role.
+func (service *Service) UpdateRole(ID portainer.RoleID, role *portainer.Role) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, role)
+}

api/bolt/schedule/schedule.go

@@ -0,0 +1,129 @@
+package schedule
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "schedules"
+)
+
+// Service represents a service for managing schedule data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// Schedule returns a schedule by ID.
+func (service *Service) Schedule(ID portainer.ScheduleID) (*portainer.Schedule, error) {
+	var schedule portainer.Schedule
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &schedule)
+	if err != nil {
+		return nil, err
+	}
+
+	return &schedule, nil
+}
+
+// UpdateSchedule updates a schedule.
+func (service *Service) UpdateSchedule(ID portainer.ScheduleID, schedule *portainer.Schedule) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, schedule)
+}
+
+// DeleteSchedule deletes a schedule.
+func (service *Service) DeleteSchedule(ID portainer.ScheduleID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}
+
+// Schedules return a array containing all the schedules.
+func (service *Service) Schedules() ([]portainer.Schedule, error) {
+	var schedules = make([]portainer.Schedule, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var schedule portainer.Schedule
+			err := internal.UnmarshalObject(v, &schedule)
+			if err != nil {
+				return err
+			}
+			schedules = append(schedules, schedule)
+		}
+
+		return nil
+	})
+
+	return schedules, err
+}
+
+// SchedulesByJobType return a array containing all the schedules
+// with the specified JobType.
+func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portainer.Schedule, error) {
+	var schedules = make([]portainer.Schedule, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var schedule portainer.Schedule
+			err := internal.UnmarshalObject(v, &schedule)
+			if err != nil {
+				return err
+			}
+			if schedule.JobType == jobType {
+				schedules = append(schedules, schedule)
+			}
+		}
+
+		return nil
+	})
+
+	return schedules, err
+}
+
+// CreateSchedule assign an ID to a new schedule and saves it.
+func (service *Service) CreateSchedule(schedule *portainer.Schedule) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		// We manually manage sequences for schedules
+		err := bucket.SetSequence(uint64(schedule.ID))
+		if err != nil {
+			return err
+		}
+
+		data, err := internal.MarshalObject(schedule)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(schedule.ID)), data)
+	})
+}
+
+// GetNextIdentifier returns the next identifier for a schedule.
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.connection, BucketName)
+}

api/bolt/services.go

@@ -0,0 +1,282 @@
+package bolt
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/customtemplate"
+	"github.com/portainer/portainer/api/bolt/dockerhub"
+	"github.com/portainer/portainer/api/bolt/edgegroup"
+	"github.com/portainer/portainer/api/bolt/edgejob"
+	"github.com/portainer/portainer/api/bolt/edgestack"
+	"github.com/portainer/portainer/api/bolt/endpoint"
+	"github.com/portainer/portainer/api/bolt/endpointgroup"
+	"github.com/portainer/portainer/api/bolt/endpointrelation"
+	"github.com/portainer/portainer/api/bolt/extension"
+	"github.com/portainer/portainer/api/bolt/helmuserrepository"
+	"github.com/portainer/portainer/api/bolt/registry"
+	"github.com/portainer/portainer/api/bolt/resourcecontrol"
+	"github.com/portainer/portainer/api/bolt/role"
+	"github.com/portainer/portainer/api/bolt/schedule"
+	"github.com/portainer/portainer/api/bolt/settings"
+	"github.com/portainer/portainer/api/bolt/ssl"
+	"github.com/portainer/portainer/api/bolt/stack"
+	"github.com/portainer/portainer/api/bolt/tag"
+	"github.com/portainer/portainer/api/bolt/team"
+	"github.com/portainer/portainer/api/bolt/teammembership"
+	"github.com/portainer/portainer/api/bolt/tunnelserver"
+	"github.com/portainer/portainer/api/bolt/user"
+	"github.com/portainer/portainer/api/bolt/version"
+	"github.com/portainer/portainer/api/bolt/webhook"
+)
+
+func (store *Store) initServices() error {
+	authorizationsetService, err := role.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.RoleService = authorizationsetService
+
+	customTemplateService, err := customtemplate.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.CustomTemplateService = customTemplateService
+
+	dockerhubService, err := dockerhub.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.DockerHubService = dockerhubService
+
+	edgeStackService, err := edgestack.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EdgeStackService = edgeStackService
+
+	edgeGroupService, err := edgegroup.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EdgeGroupService = edgeGroupService
+
+	edgeJobService, err := edgejob.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EdgeJobService = edgeJobService
+
+	endpointgroupService, err := endpointgroup.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EndpointGroupService = endpointgroupService
+
+	endpointService, err := endpoint.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EndpointService = endpointService
+
+	endpointRelationService, err := endpointrelation.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.EndpointRelationService = endpointRelationService
+
+	extensionService, err := extension.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.ExtensionService = extensionService
+
+	helmUserRepositoryService, err := helmuserrepository.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.HelmUserRepositoryService = helmUserRepositoryService
+
+	registryService, err := registry.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.RegistryService = registryService
+
+	resourcecontrolService, err := resourcecontrol.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.ResourceControlService = resourcecontrolService
+
+	settingsService, err := settings.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.SettingsService = settingsService
+
+	sslSettingsService, err := ssl.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.SSLSettingsService = sslSettingsService
+
+	stackService, err := stack.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.StackService = stackService
+
+	tagService, err := tag.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.TagService = tagService
+
+	teammembershipService, err := teammembership.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.TeamMembershipService = teammembershipService
+
+	teamService, err := team.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.TeamService = teamService
+
+	tunnelServerService, err := tunnelserver.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.TunnelServerService = tunnelServerService
+
+	userService, err := user.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.UserService = userService
+
+	versionService, err := version.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.VersionService = versionService
+
+	webhookService, err := webhook.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.WebhookService = webhookService
+
+	scheduleService, err := schedule.NewService(store.connection)
+	if err != nil {
+		return err
+	}
+	store.ScheduleService = scheduleService
+
+	return nil
+}
+
+// CustomTemplate gives access to the CustomTemplate data management layer
+func (store *Store) CustomTemplate() portainer.CustomTemplateService {
+	return store.CustomTemplateService
+}
+
+// EdgeGroup gives access to the EdgeGroup data management layer
+func (store *Store) EdgeGroup() portainer.EdgeGroupService {
+	return store.EdgeGroupService
+}
+
+// EdgeJob gives access to the EdgeJob data management layer
+func (store *Store) EdgeJob() portainer.EdgeJobService {
+	return store.EdgeJobService
+}
+
+// EdgeStack gives access to the EdgeStack data management layer
+func (store *Store) EdgeStack() portainer.EdgeStackService {
+	return store.EdgeStackService
+}
+
+// Environment(Endpoint) gives access to the Environment(Endpoint) data management layer
+func (store *Store) Endpoint() portainer.EndpointService {
+	return store.EndpointService
+}
+
+// EndpointGroup gives access to the EndpointGroup data management layer
+func (store *Store) EndpointGroup() portainer.EndpointGroupService {
+	return store.EndpointGroupService
+}
+
+// EndpointRelation gives access to the EndpointRelation data management layer
+func (store *Store) EndpointRelation() portainer.EndpointRelationService {
+	return store.EndpointRelationService
+}
+
+// HelmUserRepository access the helm user repository settings
+func (store *Store) HelmUserRepository() portainer.HelmUserRepositoryService {
+	return store.HelmUserRepositoryService
+}
+
+// Registry gives access to the Registry data management layer
+func (store *Store) Registry() portainer.RegistryService {
+	return store.RegistryService
+}
+
+// ResourceControl gives access to the ResourceControl data management layer
+func (store *Store) ResourceControl() portainer.ResourceControlService {
+	return store.ResourceControlService
+}
+
+// Role gives access to the Role data management layer
+func (store *Store) Role() portainer.RoleService {
+	return store.RoleService
+}
+
+// Settings gives access to the Settings data management layer
+func (store *Store) Settings() portainer.SettingsService {
+	return store.SettingsService
+}
+
+// SSLSettings gives access to the SSL Settings data management layer
+func (store *Store) SSLSettings() portainer.SSLSettingsService {
+	return store.SSLSettingsService
+}
+
+// Stack gives access to the Stack data management layer
+func (store *Store) Stack() portainer.StackService {
+	return store.StackService
+}
+
+// Tag gives access to the Tag data management layer
+func (store *Store) Tag() portainer.TagService {
+	return store.TagService
+}
+
+// TeamMembership gives access to the TeamMembership data management layer
+func (store *Store) TeamMembership() portainer.TeamMembershipService {
+	return store.TeamMembershipService
+}
+
+// Team gives access to the Team data management layer
+func (store *Store) Team() portainer.TeamService {
+	return store.TeamService
+}
+
+// TunnelServer gives access to the TunnelServer data management layer
+func (store *Store) TunnelServer() portainer.TunnelServerService {
+	return store.TunnelServerService
+}
+
+// User gives access to the User data management layer
+func (store *Store) User() portainer.UserService {
+	return store.UserService
+}
+
+// Version gives access to the Version data management layer
+func (store *Store) Version() portainer.VersionService {
+	return store.VersionService
+}
+
+// Webhook gives access to the Webhook data management layer
+func (store *Store) Webhook() portainer.WebhookService {
+	return store.WebhookService
+}

api/bolt/settings/settings.go

@@ -2,6 +2,7 @@ package settings
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 )
 
 const (
@@ -12,16 +13,12 @@ const (
 
 // Service represents a service for managing environment(endpoint) data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection portainer.Connection) (*Service, error) {
-	err := connection.SetServiceName(BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
@@ -35,7 +32,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
 func (service *Service) Settings() (*portainer.Settings, error) {
 	var settings portainer.Settings
 
-	err := service.connection.GetObject(BucketName, []byte(settingsKey), &settings)
+	err := internal.GetObject(service.connection, BucketName, []byte(settingsKey), &settings)
 	if err != nil {
 		return nil, err
 	}
@@ -45,19 +42,5 @@ func (service *Service) Settings() (*portainer.Settings, error) {
 
 // UpdateSettings persists a Settings object.
 func (service *Service) UpdateSettings(settings *portainer.Settings) error {
-	return service.connection.UpdateObject(BucketName, []byte(settingsKey), settings)
-}
-
-func (service *Service) IsFeatureFlagEnabled(feature portainer.Feature) bool {
-	settings, err := service.Settings()
-	if err != nil {
-		return false
-	}
-
-	featureFlagSetting, ok := settings.FeatureFlagSettings[feature]
-	if ok {
-		return featureFlagSetting
-	}
-
-	return false
+	return internal.UpdateObject(service.connection, BucketName, []byte(settingsKey), settings)
 }

api/bolt/ssl/ssl.go

@@ -2,6 +2,7 @@ package ssl
 
 import (
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
 )
 
 const (
@@ -12,16 +13,12 @@ const (
 
 // Service represents a service for managing ssl data.
 type Service struct {
-	connection portainer.Connection
-}
-
-func (service *Service) BucketName() string {
-	return BucketName
+	connection *internal.DbConnection
 }
 
 // NewService creates a new instance of a service.
-func NewService(connection portainer.Connection) (*Service, error) {
-	err := connection.SetServiceName(BucketName)
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
 	if err != nil {
 		return nil, err
 	}
@@ -35,7 +32,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
 func (service *Service) Settings() (*portainer.SSLSettings, error) {
 	var settings portainer.SSLSettings
 
-	err := service.connection.GetObject(BucketName, []byte(key), &settings)
+	err := internal.GetObject(service.connection, BucketName, []byte(key), &settings)
 	if err != nil {
 		return nil, err
 	}
@@ -45,5 +42,5 @@ func (service *Service) Settings() (*portainer.SSLSettings, error) {
 
 // UpdateSettings persists a SSLSettings object.
 func (service *Service) UpdateSettings(settings *portainer.SSLSettings) error {
-	return service.connection.UpdateObject(BucketName, []byte(key), settings)
+	return internal.UpdateObject(service.connection, BucketName, []byte(key), settings)
 }

api/bolt/stack/stack.go

@@ -0,0 +1,211 @@
+package stack
+
+import (
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+	pkgerrors "github.com/pkg/errors"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "stacks"
+)
+
+// Service represents a service for managing environment(endpoint) data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// Stack returns a stack object by ID.
+func (service *Service) Stack(ID portainer.StackID) (*portainer.Stack, error) {
+	var stack portainer.Stack
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &stack)
+	if err != nil {
+		return nil, err
+	}
+
+	return &stack, nil
+}
+
+// StackByName returns a stack object by name.
+func (service *Service) StackByName(name string) (*portainer.Stack, error) {
+	var stack *portainer.Stack
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var t portainer.Stack
+			err := internal.UnmarshalObject(v, &t)
+			if err != nil {
+				return err
+			}
+
+			if t.Name == name {
+				stack = &t
+				break
+			}
+		}
+
+		if stack == nil {
+			return errors.ErrObjectNotFound
+		}
+
+		return nil
+	})
+
+	return stack, err
+}
+
+// Stacks returns an array containing all the stacks.
+func (service *Service) Stacks() ([]portainer.Stack, error) {
+	var stacks = make([]portainer.Stack, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var stack portainer.Stack
+			err := internal.UnmarshalObject(v, &stack)
+			if err != nil {
+				return err
+			}
+			stacks = append(stacks, stack)
+		}
+
+		return nil
+	})
+
+	return stacks, err
+}
+
+// GetNextIdentifier returns the next identifier for a stack.
+func (service *Service) GetNextIdentifier() int {
+	return internal.GetNextIdentifier(service.connection, BucketName)
+}
+
+// CreateStack creates a new stack.
+func (service *Service) CreateStack(stack *portainer.Stack) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		// We manually manage sequences for stacks
+		err := bucket.SetSequence(uint64(stack.ID))
+		if err != nil {
+			return err
+		}
+
+		data, err := internal.MarshalObject(stack)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(stack.ID)), data)
+	})
+}
+
+// UpdateStack updates a stack.
+func (service *Service) UpdateStack(ID portainer.StackID, stack *portainer.Stack) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, stack)
+}
+
+// DeleteStack deletes a stack.
+func (service *Service) DeleteStack(ID portainer.StackID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}
+
+// StackByWebhookID returns a pointer to a stack object by webhook ID.
+// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID.
+func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
+	if id == "" {
+		return nil, pkgerrors.New("webhook ID can't be empty string")
+	}
+	var stack portainer.Stack
+	found := false
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var t struct {
+				AutoUpdate *struct {
+					WebhookID string `json:"Webhook"`
+				} `json:"AutoUpdate"`
+			}
+
+			err := internal.UnmarshalObject(v, &t)
+			if err != nil {
+				return err
+			}
+
+			if t.AutoUpdate != nil && strings.EqualFold(t.AutoUpdate.WebhookID, id) {
+				found = true
+				err := internal.UnmarshalObject(v, &stack)
+				if err != nil {
+					return err
+				}
+				break
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	if !found {
+		return nil, errors.ErrObjectNotFound
+	}
+
+	return &stack, nil
+}
+
+// RefreshableStacks returns stacks that are configured for a periodic update
+func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
+	stacks := make([]portainer.Stack, 0)
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+		cursor := bucket.Cursor()
+
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			stack := portainer.Stack{}
+			err := internal.UnmarshalObject(v, &stack)
+			if err != nil {
+				return err
+			}
+
+			if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
+				stacks = append(stacks, stack)
+			}
+		}
+
+		return nil
+	})
+
+	return stacks, err
+}

api/bolt/stack/tests/stack_test.go

@@ -4,10 +4,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/portainer/portainer/api/datastore"
-
 	"github.com/gofrs/uuid"
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt"
+	bolterrors "github.com/portainer/portainer/api/bolt/errors"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/stretchr/testify/assert"
 )
@@ -22,14 +22,14 @@ func newGuidString(t *testing.T) string {
 type stackBuilder struct {
 	t     *testing.T
 	count int
-	store *datastore.Store
+	store *bolt.Store
 }
 
 func TestService_StackByWebhookID(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	_, store, teardown := datastore.MustNewTestStore(true)
+	store, teardown := bolt.MustNewTestStore(true)
 	defer teardown()
 
 	b := stackBuilder{t: t, store: store}
@@ -48,7 +48,7 @@ func TestService_StackByWebhookID(t *testing.T) {
 	// returns nil and object not found error if there's no stack associated with the webhook
 	got, err = store.StackService.StackByWebhookID(newGuidString(t))
 	assert.Nil(t, got)
-	assert.True(t, store.IsErrObjectNotFound(err))
+	assert.ErrorIs(t, err, bolterrors.ErrObjectNotFound)
 }
 
 func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack {
@@ -77,7 +77,7 @@ func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack {
 		stack.AutoUpdate = &portainer.StackAutoUpdate{Webhook: webhookID}
 	}
 
-	err := b.store.StackService.Create(&stack)
+	err := b.store.StackService.CreateStack(&stack)
 	assert.NoError(b.t, err)
 
 	return stack
@@ -87,7 +87,7 @@ func Test_RefreshableStacks(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	_, store, teardown := datastore.MustNewTestStore(true)
+	store, teardown := bolt.MustNewTestStore(true)
 	defer teardown()
 
 	staticStack := portainer.Stack{ID: 1}
@@ -95,7 +95,7 @@ func Test_RefreshableStacks(t *testing.T) {
 	refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.StackAutoUpdate{Interval: "1m"}}
 
 	for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &refreshableStack} {
-		err := store.Stack().Create(stack)
+		err := store.Stack().CreateStack(stack)
 		assert.NoError(t, err)
 	}
 

api/bolt/tag/tag.go

@@ -0,0 +1,95 @@
+package tag
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "tags"
+)
+
+// Service represents a service for managing environment(endpoint) data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// Tags return an array containing all the tags.
+func (service *Service) Tags() ([]portainer.Tag, error) {
+	var tags = make([]portainer.Tag, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var tag portainer.Tag
+			err := internal.UnmarshalObject(v, &tag)
+			if err != nil {
+				return err
+			}
+			tags = append(tags, tag)
+		}
+
+		return nil
+	})
+
+	return tags, err
+}
+
+// Tag returns a tag by ID.
+func (service *Service) Tag(ID portainer.TagID) (*portainer.Tag, error) {
+	var tag portainer.Tag
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &tag)
+	if err != nil {
+		return nil, err
+	}
+
+	return &tag, nil
+}
+
+// CreateTag creates a new tag.
+func (service *Service) CreateTag(tag *portainer.Tag) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		tag.ID = portainer.TagID(id)
+
+		data, err := internal.MarshalObject(tag)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(tag.ID)), data)
+	})
+}
+
+// UpdateTag updates a tag.
+func (service *Service) UpdateTag(ID portainer.TagID, tag *portainer.Tag) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, tag)
+}
+
+// DeleteTag deletes a tag.
+func (service *Service) DeleteTag(ID portainer.TagID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}

api/bolt/team/team.go

@@ -0,0 +1,129 @@
+package team
+
+import (
+	"strings"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/errors"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "teams"
+)
+
+// Service represents a service for managing environment(endpoint) data.
+type Service struct {
+	connection *internal.DbConnection
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection *internal.DbConnection) (*Service, error) {
+	err := internal.CreateBucket(connection, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// Team returns a Team by ID
+func (service *Service) Team(ID portainer.TeamID) (*portainer.Team, error) {
+	var team portainer.Team
+	identifier := internal.Itob(int(ID))
+
+	err := internal.GetObject(service.connection, BucketName, identifier, &team)
+	if err != nil {
+		return nil, err
+	}
+
+	return &team, nil
+}
+
+// TeamByName returns a team by name.
+func (service *Service) TeamByName(name string) (*portainer.Team, error) {
+	var team *portainer.Team
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var t portainer.Team
+			err := internal.UnmarshalObject(v, &t)
+			if err != nil {
+				return err
+			}
+
+			if strings.EqualFold(t.Name, name) {
+				team = &t
+				break
+			}
+		}
+
+		if team == nil {
+			return errors.ErrObjectNotFound
+		}
+
+		return nil
+	})
+
+	return team, err
+}
+
+// Teams return an array containing all the teams.
+func (service *Service) Teams() ([]portainer.Team, error) {
+	var teams = make([]portainer.Team, 0)
+
+	err := service.connection.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		cursor := bucket.Cursor()
+		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
+			var team portainer.Team
+			err := internal.UnmarshalObject(v, &team)
+			if err != nil {
+				return err
+			}
+			teams = append(teams, team)
+		}
+
+		return nil
+	})
+
+	return teams, err
+}
+
+// UpdateTeam saves a Team.
+func (service *Service) UpdateTeam(ID portainer.TeamID, team *portainer.Team) error {
+	identifier := internal.Itob(int(ID))
+	return internal.UpdateObject(service.connection, BucketName, identifier, team)
+}
+
+// CreateTeam creates a new Team.
+func (service *Service) CreateTeam(team *portainer.Team) error {
+	return service.connection.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(BucketName))
+
+		id, _ := bucket.NextSequence()
+		team.ID = portainer.TeamID(id)
+
+		data, err := internal.MarshalObject(team)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(internal.Itob(int(team.ID)), data)
+	})
+}
+
+// DeleteTeam deletes a Team.
+func (service *Service) DeleteTeam(ID portainer.TeamID) error {
+	identifier := internal.Itob(int(ID))
+	return internal.DeleteObject(service.connection, BucketName, identifier)
+}