bump version to 2.16.1 (#8018)

* fix(snapshots): remove snapshots when removing endpoints EE-4527

* Fix nil pointer dereference.

Co-authored-by: andres-portainer <andres-portainer@users.noreply.github.com>

.dockerignore

@@ -1,3 +1,5 @@
 *
 !dist
 !build
+!metadata.json
+!docker-extension/build

.env.defaults

@@ -0,0 +1 @@
+PORTAINER_EDITION=CE

.eslintrc.yml

@@ -31,7 +31,12 @@ rules:
     [
       'error',
       {
-        pathGroups: [{ pattern: '@/**', group: 'internal' }, { pattern: '{Kubernetes,Portainer,Agent,Azure,Docker}/**', group: 'internal' }],
+        pathGroups:
+          [
+            { pattern: '@@/**', group: 'internal', position: 'after' },
+            { pattern: '@/**', group: 'internal' },
+            { pattern: '{Kubernetes,Portainer,Agent,Azure,Docker}/**', group: 'internal' },
+          ],
         groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'],
         pathGroupsExcludedImportTypes: ['internal'],
       },
@@ -41,6 +46,7 @@ settings:
   'import/resolver':
     alias:
       map:
+        - ['@@', './app/react/components']
         - ['@', './app']
       extensions: ['.js', '.ts', '.tsx']
 
@@ -52,6 +58,7 @@ overrides:
     parser: '@typescript-eslint/parser'
     plugins:
       - '@typescript-eslint'
+      - 'regex'
     extends:
       - airbnb
       - airbnb-typescript
@@ -68,7 +75,14 @@ overrides:
         version: 'detect'
     rules:
       import/order:
-        ['error', { pathGroups: [{ pattern: '@/**', group: 'internal' }], groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'], 'newlines-between': 'always' }]
+        [
+          'error',
+          {
+            pathGroups: [{ pattern: '@@/**', group: 'internal', position: 'after' }, { pattern: '@/**', group: 'internal' }],
+            groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'],
+            'newlines-between': 'always',
+          },
+        ]
       func-style: [error, 'declaration']
       import/prefer-default-export: off
       no-use-before-define: ['error', { functions: false }]
@@ -79,6 +93,7 @@ overrides:
       react/forbid-prop-types: off
       react/require-default-props: off
       react/no-array-index-key: off
+      no-underscore-dangle: off
       react/jsx-filename-extension: [0]
       import/no-extraneous-dependencies: ['error', { devDependencies: true }]
       '@typescript-eslint/explicit-module-boundary-types': off
@@ -87,6 +102,9 @@ overrides:
       'jsx-a11y/label-has-associated-control': ['error', { 'assert': 'either' }]
       'react/function-component-definition': ['error', { 'namedComponents': 'function-declaration' }]
       'react/jsx-no-bind': off
+      'no-await-in-loop': 'off'
+      'react/jsx-no-useless-fragment': ['error', { allowExpressions: true }]
+      'regex/invalid': ['error', [{ 'regex': 'data-feather="(.*)"', 'message': 'Please use `react-feather` package instead' }]]
   - files:
       - app/**/*.test.*
     extends:
@@ -94,3 +112,9 @@ overrides:
       - 'plugin:jest/style'
     env:
       'jest/globals': true
+    rules:
+      'react/jsx-no-constructed-context-values': off
+  - files:
+      - app/**/*.stories.*
+    rules:
+      'no-alert': off

.git-blame-ignore-revs

@@ -0,0 +1,5 @@
+# prettier
+cf5056d9c03b62d91a25c3b9127caac838695f98
+
+# prettier v2
+42e7db0ae7897d3cb72b0ea1ecf57ee2dd694169

.github/ISSUE_TEMPLATE.md

@@ -2,7 +2,7 @@
 
 Thanks for opening an issue on Portainer !
 
-Do you need help or have a question? Come chat with us on Slack http://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
+Do you need help or have a question? Come chat with us on Slack https://portainer.io/slack/ or gitter https://gitter.im/portainer/Lobby.
 
 If you are reporting a new issue, make sure that we do not have any duplicates
 already open. You can ensure this by searching the issue list for this

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -12,7 +12,7 @@ Thanks for reporting a bug for Portainer !
 
 You can find more information about Portainer support framework policy here: https://www.portainer.io/2019/04/portainer-support-policy/
 
-Do you need help or have a question? Come chat with us on Slack http://portainer.slack.com/
+Do you need help or have a question? Come chat with us on Slack https://portainer.io/slack/
 
 Before opening a new issue, make sure that we do not have any duplicates
 already open. You can ensure this by searching the issue list for this
@@ -47,7 +47,7 @@ You can see how [here](https://documentation.portainer.io/r/portainer-logs)
 - Platform (windows/linux):
 - Command used to start Portainer (`docker run -p 9443:9443 portainer/portainer`):
 - Browser:
-- Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commerical setup.
+- Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commercial setup.
 - Have you reviewed our technical documentation and knowledge base? Yes/No
 
 **Additional context**

.github/ISSUE_TEMPLATE/Custom.md

@@ -4,11 +4,11 @@ about: Ask us a question about Portainer usage or deployment
 title: ''
 labels: ''
 assignees: ''
-
 ---
+
 Before you start, we need a little bit more information from you:
 
-Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commerical setup.
+Use Case (delete as appropriate): Using Portainer at Home, Using Portainer in a Commercial setup.
 
 Have you reviewed our technical documentation and knowledge base? Yes/No
 
@@ -16,7 +16,7 @@ Have you reviewed our technical documentation and knowledge base? Yes/No
 
 You can find more information about Portainer support framework policy here: https://old.portainer.io/2019/04/portainer-support-policy/
 
-Do you need help or have a question? Come chat with us on Slack http://portainer.slack.com/
+Do you need help or have a question? Come chat with us on Slack https://portainer.io/slack/
 
 Also, be sure to check our FAQ and documentation first: https://documentation.portainer.io/
 -->

.github/ISSUE_TEMPLATE/Feature_request.md

@@ -4,14 +4,13 @@ about: Suggest a feature/enhancement that should be added in Portainer
 title: ''
 labels: ''
 assignees: ''
-
 ---
 
 <!--
 
 Thanks for opening a feature request for Portainer !
 
-Do you need help or have a question? Come chat with us on Slack http://portainer.slack.com/
+Do you need help or have a question? Come chat with us on Slack https://portainer.io/slack/
 
 Before opening a new issue, make sure that we do not have any duplicates
 already open. You can ensure this by searching the issue list for this

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,4 @@
+closes #0 <!-- Github issue number (remove if unknown) -->
+closes [CE-0] <!-- Jira link number (remove if unknown). Please also add the same [CE-XXX] at the back of the PR title -->
+
+### Changes:

.github/workflows/lint.yml

@@ -0,0 +1,38 @@
+name: Lint
+
+on:
+  push:
+    branches:
+      - master
+      - develop
+      - release/*
+  pull_request:
+    branches:
+      - master
+      - develop
+      - release/*
+
+jobs:
+  run-linters:
+    name: Run linters
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          cache: 'yarn'
+      - run: yarn --frozen-lockfile
+
+      - name: Run linters
+        uses: wearerequired/lint-action@v1
+        with:
+          eslint: true
+          eslint_extensions: ts,tsx,js,jsx
+          prettier: true
+          prettier_dir: app/
+          gofmt: true
+          gofmt_dir: api/
+      - name: Typecheck
+        uses: icrawl/action-tsc@v1

.github/workflows/nightly-security-scan.yml

@@ -0,0 +1,230 @@
+name: Nightly Code Security Scan
+
+on: 
+  schedule:
+    - cron: '0 8 * * *'
+  workflow_dispatch:
+    
+jobs:
+  client-dependencies:
+    name: Client dependency check
+    runs-on: ubuntu-latest
+    if: >- # only run for develop branch
+      github.ref == 'refs/heads/develop' 
+    outputs:
+      js: ${{ steps.set-matrix.outputs.js_result }}
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Run Snyk to check for vulnerabilities
+        uses: snyk/actions/node@master
+        continue-on-error: true # To make sure that artifact upload gets called
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          json: true
+
+      - name: Upload js security scan result as artifact 
+        uses: actions/upload-artifact@v3
+        with:
+          name: js-security-scan-develop-result
+          path: snyk.json
+
+      - name: Export scan result to html file 
+        run: | 
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=table -export -export-filename="/data/js-result")
+
+      - name: Upload js result html file
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-js-result-${{github.run_id}}
+          path: js-result.html
+
+      - name: Analyse the js result
+        id: set-matrix
+        run: | 
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=matrix)
+          echo "::set-output name=js_result::${result}"
+
+  server-dependencies:
+    name: Server dependency check
+    runs-on: ubuntu-latest
+    if: >- # only run for develop branch
+      github.ref == 'refs/heads/develop' 
+    outputs:
+      go: ${{ steps.set-matrix.outputs.go_result }}
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Download go modules
+        run: cd ./api && go get -t -v -d ./...
+
+      - name: Run Snyk to check for vulnerabilities
+        uses: snyk/actions/golang@master
+        continue-on-error: true # To make sure that artifact upload gets called
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          args: --file=./api/go.mod
+          json: true
+
+      - name: Upload go security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: go-security-scan-develop-result
+          path: snyk.json
+
+      - name: Export scan result to html file 
+        run: | 
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=table -export -export-filename="/data/go-result")
+
+      - name: Upload go result html file
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-go-result-${{github.run_id}}
+          path: go-result.html
+
+      - name: Analyse the go result
+        id: set-matrix
+        run: | 
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=snyk -path="/data/snyk.json" -output-type=matrix)
+          echo "::set-output name=go_result::${result}"
+
+  image-vulnerability:
+    name: Build docker image and Image vulnerability check
+    runs-on: ubuntu-latest
+    if: >-
+      github.ref == 'refs/heads/develop'
+    outputs:
+      image: ${{ steps.set-matrix.outputs.image_result }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+
+      - name: Use golang 1.18
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - name: Use Node.js 12.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+
+      - name: Install packages and build
+        run: yarn install && yarn build
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: build/linux/Dockerfile
+          tags: trivy-portainer:${{ github.sha }}
+          outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
+
+      - name: Load docker image
+        run: |
+          docker load --input /tmp/trivy-portainer-image.tar
+
+      - name: Run Trivy vulnerability scanner
+        uses: docker://docker.io/aquasec/trivy:latest
+        continue-on-error: true 
+        with:
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}  
+
+      - name: Upload image security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: image-security-scan-develop-result
+          path: image-trivy.json
+
+      - name: Export scan result to html file 
+        run: | 
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=trivy -path="/data/image-trivy.json" -output-type=table -export -export-filename="/data/image-result")
+
+      - name: Upload go result html file
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-image-result-${{github.run_id}}
+          path: image-result.html
+
+      - name: Analyse the trivy result
+        id: set-matrix
+        run: | 
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 summary -report-type=trivy -path="/data/image-trivy.json" -output-type=matrix)
+          echo "::set-output name=image_result::${result}"
+
+  result-analysis:
+    name: Analyse scan result
+    needs: [client-dependencies, server-dependencies, image-vulnerability]
+    runs-on: ubuntu-latest
+    if: >-
+      github.ref == 'refs/heads/develop'
+    strategy:
+      matrix: 
+        js: ${{fromJson(needs.client-dependencies.outputs.js)}}
+        go: ${{fromJson(needs.server-dependencies.outputs.go)}}
+        image: ${{fromJson(needs.image-vulnerability.outputs.image)}}
+    steps:
+      - name: Display the results of js, go and image
+        run: |
+          echo ${{ matrix.js.status }}
+          echo ${{ matrix.go.status }}
+          echo ${{ matrix.image.status }}
+          echo ${{ matrix.js.summary }}
+          echo ${{ matrix.go.summary }}
+          echo ${{ matrix.image.summary }}
+
+      - name: Send Slack message
+        if: >- 
+          matrix.js.status == 'failure' ||
+          matrix.go.status == 'failure' || 
+          matrix.image.status == 'failure'
+        uses: slackapi/slack-github-action@v1.18.0
+        with:
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "section",
+                  "text": {
+                    "type": "mrkdwn",
+                    "text": "Code Scanning Result (*${{ github.repository }}*)\n*<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Actions Workflow URL>*"
+                  }
+                }
+              ],
+              "attachments": [
+                {
+                  "color": "#FF0000",
+                  "blocks": [
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "*JS dependency check*: *${{ matrix.js.status }}*\n${{ matrix.js.summary }}"
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "*Go dependency check*: *${{ matrix.go.status }}*\n${{ matrix.go.summary }}"
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "*Image vulnerability check*: *${{ matrix.image.status }}*\n${{ matrix.image.summary }}\n"
+                      }
+                    }
+                  ]
+                }
+              ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SECURITY_SLACK_WEBHOOK_URL }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

.github/workflows/pr-security.yml

@@ -0,0 +1,233 @@
+name: PR Code Security Scan
+
+on:
+  pull_request_review:
+    types:
+      - submitted
+      - edited
+    paths:
+      - 'package.json'
+      - 'api/go.mod'
+      - 'gruntfile.js'
+      - 'build/linux/Dockerfile'
+      - 'build/linux/alpine.Dockerfile'
+      - 'build/windows/Dockerfile'
+    
+jobs:
+  client-dependencies:
+    name: Client dependency check
+    runs-on: ubuntu-latest
+    if: >-
+      github.event.pull_request &&
+      github.event.review.body == '/scan'
+    outputs:
+      jsdiff: ${{ steps.set-diff-matrix.outputs.js_diff_result }}
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Run Snyk to check for vulnerabilities
+        uses: snyk/actions/node@master
+        continue-on-error: true # To make sure that artifact upload gets called
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          json: true
+
+      - name: Upload js security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: js-security-scan-feat-result
+          path: snyk.json
+
+      - name: Download artifacts from develop branch
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mv ./snyk.json ./js-snyk-feature.json
+          (gh run download -n js-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
+          if [[ -e ./snyk.json ]]; then
+            mv ./snyk.json ./js-snyk-develop.json
+          else
+            echo "null" > ./js-snyk-develop.json
+          fi
+
+      - name: Export scan result to html file 
+        run: | 
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/js-snyk-feature.json" -compare-to="/data/js-snyk-develop.json" -output-type=table -export -export-filename="/data/js-result")
+
+      - name: Upload js result html file
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-js-result-compare-to-develop-${{github.run_id}}
+          path: js-result.html
+
+      - name: Analyse the js diff result
+        id: set-diff-matrix
+        run: | 
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/js-snyk-feature.json" -compare-to="./data/js-snyk-develop.json" -output-type=matrix)
+          echo "::set-output name=js_diff_result::${result}"
+
+  server-dependencies:
+    name: Server dependency check
+    runs-on: ubuntu-latest
+    if: >-
+      github.event.pull_request &&
+      github.event.review.body == '/scan'
+    outputs:
+      godiff: ${{ steps.set-diff-matrix.outputs.go_diff_result }}
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Download go modules
+        run: cd ./api && go get -t -v -d ./...
+
+      - name: Run Snyk to check for vulnerabilities
+        uses: snyk/actions/golang@master
+        continue-on-error: true # To make sure that artifact upload gets called
+        env:
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        with:
+          args: --file=./api/go.mod
+          json: true
+
+      - name: Upload go security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: go-security-scan-feature-result
+          path: snyk.json
+
+      - name: Download artifacts from develop branch
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mv ./snyk.json ./go-snyk-feature.json
+          (gh run download -n go-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
+          if [[ -e ./snyk.json ]]; then
+            mv ./snyk.json ./go-snyk-develop.json
+          else
+            echo "null" > ./go-snyk-develop.json
+          fi
+
+      - name: Export scan result to html file 
+        run: | 
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/go-snyk-feature.json" -compare-to="/data/go-snyk-develop.json" -output-type=table -export -export-filename="/data/go-result")
+
+      - name: Upload go result html file
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-go-result-compare-to-develop-${{github.run_id}}
+          path: go-result.html
+
+      - name: Analyse the go diff result
+        id: set-diff-matrix
+        run: | 
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=snyk -path="/data/go-snyk-feature.json" -compare-to="/data/go-snyk-develop.json" -output-type=matrix)
+          echo "::set-output name=go_diff_result::${result}"
+
+  image-vulnerability:
+    name: Build docker image and Image vulnerability check
+    runs-on: ubuntu-latest
+    if: >-
+      github.event.pull_request &&
+      github.event.review.body == '/scan'
+    outputs:
+      imagediff: ${{ steps.set-diff-matrix.outputs.image_diff_result }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@master
+
+      - name: Use golang 1.18
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - name: Use Node.js 12.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+
+      - name: Install packages and build
+        run: yarn install && yarn build
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: build/linux/Dockerfile
+          tags: trivy-portainer:${{ github.sha }}
+          outputs: type=docker,dest=/tmp/trivy-portainer-image.tar
+
+      - name: Load docker image
+        run: |
+          docker load --input /tmp/trivy-portainer-image.tar
+
+      - name: Run Trivy vulnerability scanner
+        uses: docker://docker.io/aquasec/trivy:latest
+        continue-on-error: true 
+        with:
+          args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress trivy-portainer:${{ github.sha }}  
+
+      - name: Upload image security scan result as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: image-security-scan-feature-result
+          path: image-trivy.json
+
+      - name: Download artifacts from develop branch
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mv ./image-trivy.json ./image-trivy-feature.json
+          (gh run download -n image-security-scan-develop-result -R ${{ github.repository }} 2>&1 >/dev/null) || :
+          if [[ -e ./image-trivy.json ]]; then
+            mv ./image-trivy.json ./image-trivy-develop.json
+          else
+            echo "null" > ./image-trivy-develop.json
+          fi
+
+      - name: Export scan result to html file 
+        run: | 
+          $(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=trivy -path="/data/image-trivy-feature.json" -compare-to="/data/image-trivy-develop.json" -output-type=table -export -export-filename="/data/image-result")
+
+      - name: Upload image result html file
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-image-result-compare-to-develop-${{github.run_id}}
+          path: image-result.html
+
+      - name: Analyse the image diff result
+        id: set-diff-matrix
+        run: | 
+          result=$(docker run --rm -v ${{ github.workspace }}:/data oscarzhou/scan-report:0.1.8 diff -report-type=trivy -path="/data/image-trivy-feature.json" -compare-to="./data/image-trivy-develop.json" -output-type=matrix)
+          echo "::set-output name=image_diff_result::${result}"
+
+  result-analysis:
+    name: Analyse scan result compared to develop
+    needs: [client-dependencies, server-dependencies, image-vulnerability]
+    runs-on: ubuntu-latest
+    if: >-
+      github.event.pull_request &&
+      github.event.review.body == '/scan'
+    strategy:
+      matrix: 
+        jsdiff: ${{fromJson(needs.client-dependencies.outputs.jsdiff)}}
+        godiff: ${{fromJson(needs.server-dependencies.outputs.godiff)}}
+        imagediff: ${{fromJson(needs.image-vulnerability.outputs.imagediff)}}
+    steps:
+
+      - name: Check job status of diff result
+        if: >-
+          matrix.jsdiff.status == 'failure' ||
+          matrix.godiff.status == 'failure' ||
+          matrix.imagediff.status == 'failure' 
+        run: |
+          echo ${{ matrix.jsdiff.status }}
+          echo ${{ matrix.godiff.status }}
+          echo ${{ matrix.imagediff.status }}
+          echo ${{ matrix.jsdiff.summary }}
+          echo ${{ matrix.godiff.summary }}
+          echo ${{ matrix.imagediff.summary }}
+          exit 1

.github/workflows/test.yaml

@@ -0,0 +1,29 @@
+name: Test
+on: push
+jobs:
+  test-client:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          cache: 'yarn'
+      - run: yarn --frozen-lockfile
+
+      - name: Run tests
+        run: yarn test:client
+  # test-server:
+  #   runs-on: ubuntu-latest
+  #   env:
+  #     GOPRIVATE: "github.com/portainer"
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - uses: actions/setup-go@v3
+  #       with:
+  #         go-version: '1.18'
+  #     - name: Run tests
+  #       run: |
+  #         cd api
+  #         go test ./...

.gitignore

@@ -7,6 +7,7 @@ storybook-static
 .tmp
 **/.vscode/settings.json
 **/.vscode/tasks.json
+.vscode
 *.DS_Store
 
 .eslintcache

.prettierignore

@@ -1 +1,2 @@
-dist
+dist
+api/datastore/test_data

.prettierrc

@@ -4,21 +4,16 @@
   "htmlWhitespaceSensitivity": "strict",
   "overrides": [
     {
-      "files": [
-        "*.html"
-      ],
+      "files": ["*.html"],
       "options": {
         "parser": "angular"
       }
     },
     {
-      "files": [
-        "*.{j,t}sx",
-        "*.ts"
-      ],
+      "files": ["*.{j,t}sx", "*.ts"],
       "options": {
         "printWidth": 80
       }
     }
   ]
-}
+}

.storybook/main.js

@@ -16,6 +16,9 @@ module.exports = {
             exportLocalsConvention: 'camelCaseOnly',
           },
         },
+        postcssLoaderOptions: {
+          implementation: require('postcss'),
+        },
       },
     },
   ],
@@ -31,4 +34,5 @@ module.exports = {
   core: {
     builder: 'webpack5',
   },
+  staticDirs: ['./public'],
 };

.storybook/preview.js

@@ -1,6 +1,23 @@
 import '../app/assets/css';
 
 import { pushStateLocationPlugin, UIRouter } from '@uirouter/react';
+import { initialize as initMSW, mswDecorator } from 'msw-storybook-addon';
+import { handlers } from '@/setup-tests/server-handlers';
+import { QueryClient, QueryClientProvider } from 'react-query';
+
+// Initialize MSW
+initMSW({
+  onUnhandledRequest: ({ method, url }) => {
+    if (url.pathname.startsWith('/api')) {
+      console.error(`Unhandled ${method} request to ${url}.
+
+        This exception has been only logged in the console, however, it's strongly recommended to resolve this error as you don't want unmocked data in Storybook stories.
+
+        If you wish to mock an error response, please refer to this guide: https://mswjs.io/docs/recipes/mocking-error-responses
+      `);
+    }
+  },
+});
 
 export const parameters = {
   actions: { argTypesRegex: '^on[A-Z].*' },
@@ -10,12 +27,22 @@ export const parameters = {
       date: /Date$/,
     },
   },
+  msw: {
+    handlers,
+  },
 };
 
+const testQueryClient = new QueryClient({
+  defaultOptions: { queries: { retry: false } },
+});
+
 export const decorators = [
   (Story) => (
-    <UIRouter plugins={[pushStateLocationPlugin]}>
-      <Story />
-    </UIRouter>
+    <QueryClientProvider client={testQueryClient}>
+      <UIRouter plugins={[pushStateLocationPlugin]}>
+        <Story />
+      </UIRouter>
+    </QueryClientProvider>
   ),
+  mswDecorator,
 ];

.storybook/public/mockServiceWorker.js

@@ -0,0 +1,328 @@
+/* eslint-disable */
+/* tslint:disable */
+
+/**
+ * Mock Service Worker (0.36.3).
+ * @see https://github.com/mswjs/msw
+ * - Please do NOT modify this file.
+ * - Please do NOT serve this file on production.
+ */
+
+const INTEGRITY_CHECKSUM = '02f4ad4a2797f85668baf196e553d929';
+const bypassHeaderName = 'x-msw-bypass';
+const activeClientIds = new Set();
+
+self.addEventListener('install', function () {
+  return self.skipWaiting();
+});
+
+self.addEventListener('activate', async function (event) {
+  return self.clients.claim();
+});
+
+self.addEventListener('message', async function (event) {
+  const clientId = event.source.id;
+
+  if (!clientId || !self.clients) {
+    return;
+  }
+
+  const client = await self.clients.get(clientId);
+
+  if (!client) {
+    return;
+  }
+
+  const allClients = await self.clients.matchAll();
+
+  switch (event.data) {
+    case 'KEEPALIVE_REQUEST': {
+      sendToClient(client, {
+        type: 'KEEPALIVE_RESPONSE',
+      });
+      break;
+    }
+
+    case 'INTEGRITY_CHECK_REQUEST': {
+      sendToClient(client, {
+        type: 'INTEGRITY_CHECK_RESPONSE',
+        payload: INTEGRITY_CHECKSUM,
+      });
+      break;
+    }
+
+    case 'MOCK_ACTIVATE': {
+      activeClientIds.add(clientId);
+
+      sendToClient(client, {
+        type: 'MOCKING_ENABLED',
+        payload: true,
+      });
+      break;
+    }
+
+    case 'MOCK_DEACTIVATE': {
+      activeClientIds.delete(clientId);
+      break;
+    }
+
+    case 'CLIENT_CLOSED': {
+      activeClientIds.delete(clientId);
+
+      const remainingClients = allClients.filter((client) => {
+        return client.id !== clientId;
+      });
+
+      // Unregister itself when there are no more clients
+      if (remainingClients.length === 0) {
+        self.registration.unregister();
+      }
+
+      break;
+    }
+  }
+});
+
+// Resolve the "main" client for the given event.
+// Client that issues a request doesn't necessarily equal the client
+// that registered the worker. It's with the latter the worker should
+// communicate with during the response resolving phase.
+async function resolveMainClient(event) {
+  const client = await self.clients.get(event.clientId);
+
+  if (client.frameType === 'top-level') {
+    return client;
+  }
+
+  const allClients = await self.clients.matchAll();
+
+  return allClients
+    .filter((client) => {
+      // Get only those clients that are currently visible.
+      return client.visibilityState === 'visible';
+    })
+    .find((client) => {
+      // Find the client ID that's recorded in the
+      // set of clients that have registered the worker.
+      return activeClientIds.has(client.id);
+    });
+}
+
+async function handleRequest(event, requestId) {
+  const client = await resolveMainClient(event);
+  const response = await getResponse(event, client, requestId);
+
+  // Send back the response clone for the "response:*" life-cycle events.
+  // Ensure MSW is active and ready to handle the message, otherwise
+  // this message will pend indefinitely.
+  if (client && activeClientIds.has(client.id)) {
+    (async function () {
+      const clonedResponse = response.clone();
+      sendToClient(client, {
+        type: 'RESPONSE',
+        payload: {
+          requestId,
+          type: clonedResponse.type,
+          ok: clonedResponse.ok,
+          status: clonedResponse.status,
+          statusText: clonedResponse.statusText,
+          body: clonedResponse.body === null ? null : await clonedResponse.text(),
+          headers: serializeHeaders(clonedResponse.headers),
+          redirected: clonedResponse.redirected,
+        },
+      });
+    })();
+  }
+
+  return response;
+}
+
+async function getResponse(event, client, requestId) {
+  const { request } = event;
+  const requestClone = request.clone();
+  const getOriginalResponse = () => fetch(requestClone);
+
+  // Bypass mocking when the request client is not active.
+  if (!client) {
+    return getOriginalResponse();
+  }
+
+  // Bypass initial page load requests (i.e. static assets).
+  // The absence of the immediate/parent client in the map of the active clients
+  // means that MSW hasn't dispatched the "MOCK_ACTIVATE" event yet
+  // and is not ready to handle requests.
+  if (!activeClientIds.has(client.id)) {
+    return await getOriginalResponse();
+  }
+
+  // Bypass requests with the explicit bypass header
+  if (requestClone.headers.get(bypassHeaderName) === 'true') {
+    const cleanRequestHeaders = serializeHeaders(requestClone.headers);
+
+    // Remove the bypass header to comply with the CORS preflight check.
+    delete cleanRequestHeaders[bypassHeaderName];
+
+    const originalRequest = new Request(requestClone, {
+      headers: new Headers(cleanRequestHeaders),
+    });
+
+    return fetch(originalRequest);
+  }
+
+  // Send the request to the client-side MSW.
+  const reqHeaders = serializeHeaders(request.headers);
+  const body = await request.text();
+
+  const clientMessage = await sendToClient(client, {
+    type: 'REQUEST',
+    payload: {
+      id: requestId,
+      url: request.url,
+      method: request.method,
+      headers: reqHeaders,
+      cache: request.cache,
+      mode: request.mode,
+      credentials: request.credentials,
+      destination: request.destination,
+      integrity: request.integrity,
+      redirect: request.redirect,
+      referrer: request.referrer,
+      referrerPolicy: request.referrerPolicy,
+      body,
+      bodyUsed: request.bodyUsed,
+      keepalive: request.keepalive,
+    },
+  });
+
+  switch (clientMessage.type) {
+    case 'MOCK_SUCCESS': {
+      return delayPromise(() => respondWithMock(clientMessage), clientMessage.payload.delay);
+    }
+
+    case 'MOCK_NOT_FOUND': {
+      return getOriginalResponse();
+    }
+
+    case 'NETWORK_ERROR': {
+      const { name, message } = clientMessage.payload;
+      const networkError = new Error(message);
+      networkError.name = name;
+
+      // Rejecting a request Promise emulates a network error.
+      throw networkError;
+    }
+
+    case 'INTERNAL_ERROR': {
+      const parsedBody = JSON.parse(clientMessage.payload.body);
+
+      console.error(
+        `\
+[MSW] Uncaught exception in the request handler for "%s %s":
+
+${parsedBody.location}
+
+This exception has been gracefully handled as a 500 response, however, it's strongly recommended to resolve this error, as it indicates a mistake in your code. If you wish to mock an error response, please see this guide: https://mswjs.io/docs/recipes/mocking-error-responses\
+`,
+        request.method,
+        request.url
+      );
+
+      return respondWithMock(clientMessage);
+    }
+  }
+
+  return getOriginalResponse();
+}
+
+self.addEventListener('fetch', function (event) {
+  const { request } = event;
+  const accept = request.headers.get('accept') || '';
+
+  // Bypass server-sent events.
+  if (accept.includes('text/event-stream')) {
+    return;
+  }
+
+  // Bypass navigation requests.
+  if (request.mode === 'navigate') {
+    return;
+  }
+
+  // Opening the DevTools triggers the "only-if-cached" request
+  // that cannot be handled by the worker. Bypass such requests.
+  if (request.cache === 'only-if-cached' && request.mode !== 'same-origin') {
+    return;
+  }
+
+  // Bypass all requests when there are no active clients.
+  // Prevents the self-unregistered worked from handling requests
+  // after it's been deleted (still remains active until the next reload).
+  if (activeClientIds.size === 0) {
+    return;
+  }
+
+  const requestId = uuidv4();
+
+  return event.respondWith(
+    handleRequest(event, requestId).catch((error) => {
+      if (error.name === 'NetworkError') {
+        console.warn('[MSW] Successfully emulated a network error for the "%s %s" request.', request.method, request.url);
+        return;
+      }
+
+      // At this point, any exception indicates an issue with the original request/response.
+      console.error(
+        `\
+[MSW] Caught an exception from the "%s %s" request (%s). This is probably not a problem with Mock Service Worker. There is likely an additional logging output above.`,
+        request.method,
+        request.url,
+        `${error.name}: ${error.message}`
+      );
+    })
+  );
+});
+
+function serializeHeaders(headers) {
+  const reqHeaders = {};
+  headers.forEach((value, name) => {
+    reqHeaders[name] = reqHeaders[name] ? [].concat(reqHeaders[name]).concat(value) : value;
+  });
+  return reqHeaders;
+}
+
+function sendToClient(client, message) {
+  return new Promise((resolve, reject) => {
+    const channel = new MessageChannel();
+
+    channel.port1.onmessage = (event) => {
+      if (event.data && event.data.error) {
+        return reject(event.data.error);
+      }
+
+      resolve(event.data);
+    };
+
+    client.postMessage(JSON.stringify(message), [channel.port2]);
+  });
+}
+
+function delayPromise(cb, duration) {
+  return new Promise((resolve) => {
+    setTimeout(() => resolve(cb()), duration);
+  });
+}
+
+function respondWithMock(clientMessage) {
+  return new Response(clientMessage.payload.body, {
+    ...clientMessage.payload,
+    headers: clientMessage.payload.headers,
+  });
+}
+
+function uuidv4() {
+  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
+    const r = (Math.random() * 16) | 0;
+    const v = c == 'x' ? r : (r & 0x3) | 0x8;
+    return v.toString(16);
+  });
+}

.vscode.example/launch.json

@@ -9,7 +9,7 @@
       "type": "go",
       "request": "launch",
       "mode": "debug",
-      "program": "${workspaceRoot}/api/cmd/portainer/main.go",
+      "program": "${workspaceRoot}/api/cmd/portainer",
       "cwd": "${workspaceRoot}",
       "env": {},
       "showLog": true,

.vscode.example/settings.json

@@ -1,4 +1,8 @@
 {
   "go.lintTool": "golangci-lint",
-  "go.lintFlags": ["--fast", "-E", "exportloopref"]
+  "go.lintFlags": ["--fast", "-E", "exportloopref"],
+  "gopls": {
+    "build.expandWorkspaceToModule": false
+  },
+  "gitlens.advanced.blame.customArguments": ["--ignore-revs-file", ".git-blame-ignore-revs"]
 }

README.md

@@ -12,21 +12,15 @@ Portainer consists of a single container that can run on any cluster. It can be
 - [Take5 – get 5 free nodes of Portainer Business for as long as you want them](https://portainer.io/pricing/take5)
 - [Portainer BE install guide](https://install.portainer.io)
 
-## Demo
-
-You can try out the public demo instance: http://demo.portainer.io/ (login with the username **admin** and the password **tryportainer**).
-
-Please note that the public demo cluster is **reset every 15min**.
-
 ## Latest Version
 
 Portainer CE is updated regularly. We aim to do an update release every couple of months.
 
-**The latest version of Portainer is 2.9.x**. Portainer is on version 2, the second number denotes the month of release.
+[![latest version](https://img.shields.io/github/v/release/portainer/portainer?color=%2344cc11&label=Latest%20release&style=for-the-badge)](https://github.com/portainer/portainer/releases/latest)
 
 ## Getting started
 
-- [Deploy Portainer](https://docs.portainer.io/v/ce-2.9/start/install)
+- [Deploy Portainer](https://docs.portainer.io/start/install)
 - [Documentation](https://documentation.portainer.io)
 - [Contribute to the project](https://documentation.portainer.io/contributing/instructions/)
 
@@ -42,10 +36,10 @@ View [this](https://www.portainer.io/products) table to see all of the Portainer
 
 Portainer CE is an open source project and is supported by the community. You can buy a supported version of Portainer at portainer.io
 
-Learn more about Portainers community support channels [here.](https://www.portainer.io/community_help)
+Learn more about Portainer's community support channels [here.](https://www.portainer.io/community_help)
 
 - Issues: https://github.com/portainer/portainer/issues
-- Slack (chat): [https://portainer.slack.com/](https://join.slack.com/t/portainer/shared_invite/zt-txh3ljab-52QHTyjCqbe5RibC2lcjKA)
+- Slack (chat): [https://portainer.io/slack](https://portainer.io/slack)
 
 You can join the Portainer Community by visiting community.portainer.io. This will give you advance notice of events, content and other related Portainer content.
 

api/adminmonitor/admin_monitor.go

@@ -2,57 +2,83 @@ package adminmonitor
 
 import (
 	"context"
-	"log"
+	"net/http"
+	"strings"
+	"sync"
 	"time"
 
+	httperror "github.com/portainer/libhttp/error"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
+
+	"github.com/rs/zerolog/log"
 )
 
-var logFatalf = log.Fatalf
+const RedirectReasonAdminInitTimeout string = "AdminInitTimeout"
 
 type Monitor struct {
-	timeout          time.Duration
-	datastore        dataservices.DataStore
-	shutdownCtx      context.Context
-	cancellationFunc context.CancelFunc
+	timeout           time.Duration
+	datastore         dataservices.DataStore
+	shutdownCtx       context.Context
+	cancellationFunc  context.CancelFunc
+	mu                sync.Mutex
+	adminInitDisabled bool
 }
 
-// New creates a monitor that when started will wait for the timeout duration and then shutdown the application unless it has been initialized.
+// New creates a monitor that when started will wait for the timeout duration and then sends the timeout signal to disable the application
 func New(timeout time.Duration, datastore dataservices.DataStore, shutdownCtx context.Context) *Monitor {
 	return &Monitor{
-		timeout:     timeout,
-		datastore:   datastore,
-		shutdownCtx: shutdownCtx,
+		timeout:           timeout,
+		datastore:         datastore,
+		shutdownCtx:       shutdownCtx,
+		adminInitDisabled: false,
 	}
 }
 
 // Starts starts the monitor. Active monitor could be stopped or shuttted down by cancelling the shutdown context.
 func (m *Monitor) Start() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if m.cancellationFunc != nil {
+		return
+	}
+
 	cancellationCtx, cancellationFunc := context.WithCancel(context.Background())
 	m.cancellationFunc = cancellationFunc
 
 	go func() {
-		log.Println("[DEBUG] [internal,init] [message: start initialization monitor ]")
+		log.Debug().Msg("start initialization monitor")
+
 		select {
 		case <-time.After(m.timeout):
 			initialized, err := m.WasInitialized()
 			if err != nil {
-				logFatalf("%s", err)
+				log.Fatal().Err(err).Msg("")
 			}
+
 			if !initialized {
-				logFatalf("[FATAL] [internal,init] No administrator account was created in %f mins. Shutting down the Portainer instance for security reasons", m.timeout.Minutes())
+				log.Info().Msg("the Portainer instance timed out for security purposes, to re-enable your Portainer instance, you will need to restart Portainer")
+
+				m.mu.Lock()
+				defer m.mu.Unlock()
+
+				m.adminInitDisabled = true
+				return
 			}
 		case <-cancellationCtx.Done():
-			log.Println("[DEBUG] [internal,init] [message: canceling initialization monitor]")
+			log.Debug().Msg("canceling initialization monitor")
 		case <-m.shutdownCtx.Done():
-			log.Println("[DEBUG] [internal,init] [message: shutting down initialization monitor]")
+			log.Debug().Msg("shutting down initialization monitor")
 		}
 	}()
 }
 
 // Stop stops monitor. Safe to call even if monitor wasn't started.
 func (m *Monitor) Stop() {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
 	if m.cancellationFunc == nil {
 		return
 	}
@@ -68,3 +94,25 @@ func (m *Monitor) WasInitialized() (bool, error) {
 	}
 	return len(users) > 0, nil
 }
+
+func (m *Monitor) WasInstanceDisabled() bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return m.adminInitDisabled
+}
+
+// WithRedirect checks whether administrator initialisation timeout. If so, it will return the error with redirect reason.
+// Otherwise, it will pass through the request to next
+func (m *Monitor) WithRedirect(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if m.WasInstanceDisabled() {
+			if strings.HasPrefix(r.RequestURI, "/api") && r.RequestURI != "/api/status" && r.RequestURI != "/api/settings/public" {
+				w.Header().Set("redirect-reason", RedirectReasonAdminInitTimeout)
+				httperror.WriteError(w, http.StatusSeeOther, "Administrator initialization timeout", nil)
+				return
+			}
+		}
+
+		next.ServeHTTP(w, r)
+	})
+}

api/adminmonitor/admin_monitor_test.go

@@ -21,6 +21,18 @@ func Test_stopCouldBeCalledMultipleTimes(t *testing.T) {
 	monitor.Stop()
 }
 
+func Test_startOrStopCouldBeCalledMultipleTimesConcurrently(t *testing.T) {
+	monitor := New(1*time.Minute, nil, context.Background())
+
+	go monitor.Start()
+	monitor.Start()
+
+	go monitor.Stop()
+	monitor.Stop()
+
+	time.Sleep(2 * time.Second)
+}
+
 func Test_canStopStartedMonitor(t *testing.T) {
 	monitor := New(1*time.Minute, nil, context.Background())
 	monitor.Start()
@@ -30,21 +42,13 @@ func Test_canStopStartedMonitor(t *testing.T) {
 	assert.Nil(t, monitor.cancellationFunc, "cancellation function should absent in stopped monitor")
 }
 
-func Test_start_shouldFatalAfterTimeout_ifNotInitialized(t *testing.T) {
+func Test_start_shouldDisableInstanceAfterTimeout_ifNotInitialized(t *testing.T) {
 	timeout := 10 * time.Millisecond
 
 	datastore := i.NewDatastore(i.WithUsers([]portainer.User{}))
-
-	var fataled bool
-	origLogFatalf := logFatalf
-	logFatalf = func(s string, v ...interface{}) { fataled = true }
-	defer func() {
-		logFatalf = origLogFatalf
-	}()
-
 	monitor := New(timeout, datastore, context.Background())
 	monitor.Start()
-	<-time.After(2 * timeout)
 
-	assert.True(t, fataled, "monitor should been timeout and fatal")
+	<-time.After(20 * timeout)
+	assert.True(t, monitor.WasInstanceDisabled(), "monitor should have been timeout and instance is disabled")
 }

api/agent/version.go

@@ -0,0 +1,71 @@
+package agent
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/internal/url"
+)
+
+// GetAgentVersionAndPlatform returns the agent version and platform
+//
+// it sends a ping to the agent and parses the version and platform from the headers
+func GetAgentVersionAndPlatform(endpointUrl string, tlsConfig *tls.Config) (portainer.AgentPlatform, string, error) {
+	httpCli := &http.Client{
+		Timeout: 3 * time.Second,
+	}
+
+	if tlsConfig != nil {
+		httpCli.Transport = &http.Transport{
+			TLSClientConfig: tlsConfig,
+		}
+	}
+
+	parsedURL, err := url.ParseURL(endpointUrl + "/ping")
+	if err != nil {
+		return 0, "", err
+	}
+
+	parsedURL.Scheme = "https"
+
+	req, err := http.NewRequest(http.MethodGet, parsedURL.String(), nil)
+	if err != nil {
+		return 0, "", err
+	}
+
+	resp, err := httpCli.Do(req)
+	if err != nil {
+		return 0, "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusNoContent {
+		return 0, "", fmt.Errorf("Failed request with status %d", resp.StatusCode)
+	}
+
+	version := resp.Header.Get(portainer.PortainerAgentHeader)
+	if version == "" {
+		return 0, "", errors.New("Version Header is missing")
+	}
+
+	agentPlatformHeader := resp.Header.Get(portainer.HTTPResponseAgentPlatform)
+	if agentPlatformHeader == "" {
+		return 0, "", errors.New("Agent Platform Header is missing")
+	}
+
+	agentPlatformNumber, err := strconv.Atoi(agentPlatformHeader)
+	if err != nil {
+		return 0, "", err
+	}
+
+	if agentPlatformNumber == 0 {
+		return 0, "", errors.New("Agent platform is invalid")
+	}
+
+	return portainer.AgentPlatform(agentPlatformNumber), version, nil
+}

api/api-description.md

@@ -50,4 +50,15 @@ Instead, it acts as a reverse-proxy to the Docker HTTP API. This means that you
 
 To do so, you can use the `/endpoints/{id}/docker` Portainer API environment(endpoint) (which is not documented below due to Swagger limitations). This environment(endpoint) has a restricted access policy so you still need to be authenticated to be able to query this environment(endpoint). Any query on this environment(endpoint) will be proxied to the Docker API of the associated environment(endpoint) (requests and responses objects are the same as documented in the Docker API).
 
+# Private Registry
+
+Using private registry, you will need to pass a based64 encoded JSON string ‘{"registryId":\<registryID value\>}’ inside the Request Header. The parameter name is "X-Registry-Auth".
+\<registryID value\> - The registry ID where the repository was created.
+
+Example:
+
+```
+eyJyZWdpc3RyeUlkIjoxfQ==
+```
+
 **NOTE**: You can find more information on how to query the Docker API in the [Docker official documentation](https://docs.docker.com/engine/api/v1.30/) as well as in [this Portainer example](https://documentation.portainer.io/api/api-examples/).

api/apikey/service_test.go

@@ -2,7 +2,6 @@ package apikey
 
 import (
 	"crypto/sha256"
-	"log"
 	"strings"
 	"testing"
 	"time"
@@ -10,6 +9,8 @@ import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/rs/zerolog/log"
 )
 
 func Test_SatisfiesAPIKeyServiceInterface(t *testing.T) {
@@ -20,7 +21,7 @@ func Test_SatisfiesAPIKeyServiceInterface(t *testing.T) {
 func Test_GenerateApiKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -74,7 +75,7 @@ func Test_GenerateApiKey(t *testing.T) {
 func Test_GetAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -94,7 +95,7 @@ func Test_GetAPIKey(t *testing.T) {
 func Test_GetAPIKeys(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -115,7 +116,7 @@ func Test_GetAPIKeys(t *testing.T) {
 func Test_GetDigestUserAndKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -151,7 +152,7 @@ func Test_GetDigestUserAndKey(t *testing.T) {
 func Test_UpdateAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -169,8 +170,8 @@ func Test_UpdateAPIKey(t *testing.T) {
 		_, apiKeyGot, err := service.GetDigestUserAndKey(apiKey.Digest)
 		is.NoError(err)
 
-		log.Println(apiKey)
-		log.Println(apiKeyGot)
+		log.Debug().Msgf("%+v", apiKey)
+		log.Debug().Msgf("%+v", apiKeyGot)
 
 		is.Equal(apiKey.LastUsed, apiKeyGot.LastUsed)
 
@@ -199,7 +200,7 @@ func Test_UpdateAPIKey(t *testing.T) {
 func Test_DeleteAPIKey(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())
@@ -240,7 +241,7 @@ func Test_DeleteAPIKey(t *testing.T) {
 func Test_InvalidateUserKeyCache(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	service := NewAPIKeyService(store.APIKeyRepository(), store.User())

api/archive/tar.go

@@ -34,3 +34,45 @@ func TarFileInBuffer(fileContent []byte, fileName string, mode int64) ([]byte, e
 
 	return buffer.Bytes(), nil
 }
+
+// tarFileInBuffer represents a tar archive buffer.
+type tarFileInBuffer struct {
+	b *bytes.Buffer
+	w *tar.Writer
+}
+
+func NewTarFileInBuffer() *tarFileInBuffer {
+	var b bytes.Buffer
+	return &tarFileInBuffer{
+		b: &b,
+		w: tar.NewWriter(&b),
+	}
+}
+
+// Put puts a single file to tar archive buffer.
+func (t *tarFileInBuffer) Put(fileContent []byte, fileName string, mode int64) error {
+	hdr := &tar.Header{
+		Name: fileName,
+		Mode: mode,
+		Size: int64(len(fileContent)),
+	}
+
+	if err := t.w.WriteHeader(hdr); err != nil {
+		return err
+	}
+
+	if _, err := t.w.Write(fileContent); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Bytes returns the archive as a byte array.
+func (t *tarFileInBuffer) Bytes() []byte {
+	return t.b.Bytes()
+}
+
+func (t *tarFileInBuffer) Close() error {
+	return t.w.Close()
+}

api/archive/targz_test.go

@@ -9,7 +9,6 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/docker/docker/pkg/ioutils"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -27,9 +26,7 @@ func listFiles(dir string) []string {
 }
 
 func Test_shouldCreateArhive(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
+	tmpdir := t.TempDir()
 	content := []byte("content")
 	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
 	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
@@ -40,9 +37,7 @@ func Test_shouldCreateArhive(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
 
-	extractionDir, _ := ioutils.TempDir("", "extract")
-	defer os.RemoveAll(extractionDir)
-
+	extractionDir := t.TempDir()
 	cmd := exec.Command("tar", "-xzf", gzPath, "-C", extractionDir)
 	err = cmd.Run()
 	if err != nil {
@@ -63,9 +58,7 @@ func Test_shouldCreateArhive(t *testing.T) {
 }
 
 func Test_shouldCreateArhiveXXXXX(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "backup")
-	defer os.RemoveAll(tmpdir)
-
+	tmpdir := t.TempDir()
 	content := []byte("content")
 	ioutil.WriteFile(path.Join(tmpdir, "outer"), content, 0600)
 	os.MkdirAll(path.Join(tmpdir, "dir"), 0700)
@@ -76,9 +69,7 @@ func Test_shouldCreateArhiveXXXXX(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, filepath.Join(tmpdir, fmt.Sprintf("%s.tar.gz", filepath.Base(tmpdir))), gzPath)
 
-	extractionDir, _ := ioutils.TempDir("", "extract")
-	defer os.RemoveAll(extractionDir)
-
+	extractionDir := t.TempDir()
 	r, _ := os.Open(gzPath)
 	ExtractTarGz(r, extractionDir)
 	if err != nil {

api/archive/zip_test.go

@@ -2,16 +2,12 @@ package archive
 
 import (
 	"github.com/stretchr/testify/assert"
-	"io/ioutil"
-	"os"
 	"path/filepath"
 	"testing"
 )
 
 func TestUnzipFile(t *testing.T) {
-	dir, err := ioutil.TempDir("", "unzip-test-")
-	assert.NoError(t, err)
-	defer os.RemoveAll(dir)
+	dir := t.TempDir()
 	/*
 		Archive structure.
 		├── 0
@@ -21,7 +17,7 @@ func TestUnzipFile(t *testing.T) {
 		└── 0.txt
 	*/
 
-	err = UnzipFile("./testdata/sample_archive.zip", dir)
+	err := UnzipFile("./testdata/sample_archive.zip", dir)
 
 	assert.NoError(t, err)
 	archiveDir := dir + "/sample_archive"

api/backup/backup.go

@@ -7,13 +7,14 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/pkg/errors"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/offlinegate"
-	"github.com/sirupsen/logrus"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
 )
 
 const rwxr__r__ os.FileMode = 0744
@@ -47,9 +48,9 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto
 
 		err := datastore.Export(exportFilename)
 		if err != nil {
-			logrus.WithError(err).Debugf("failed to export to %s", exportFilename)
+			log.Error().Err(err).Str("filename", exportFilename).Msg("failed to export")
 		} else {
-			logrus.Debugf("exported to %s", exportFilename)
+			log.Debug().Str("filename", exportFilename).Msg("file exported")
 		}
 	}
 

api/backup/restore.go

@@ -10,6 +10,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/portainer/portainer/api/archive"
 	"github.com/portainer/portainer/api/crypto"
+	"github.com/portainer/portainer/api/database/boltdb"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/filesystem"
 	"github.com/portainer/portainer/api/http/offlinegate"
@@ -65,5 +66,20 @@ func restoreFiles(srcDir string, destinationDir string) error {
 			return err
 		}
 	}
-	return nil
+
+	// TODO:  This is very boltdb module specific once again due to the filename.  Move to bolt module? Refactor for another day
+
+	// Prevent the possibility of having both databases.  Remove any default new instance
+	os.Remove(filepath.Join(destinationDir, boltdb.DatabaseFileName))
+	os.Remove(filepath.Join(destinationDir, boltdb.EncryptedDatabaseFileName))
+
+	// Now copy the database.  It'll be either portainer.db or portainer.edb
+
+	// Note: CopyPath does not return an error if the source file doesn't exist
+	err := filesystem.CopyPath(filepath.Join(srcDir, boltdb.EncryptedDatabaseFileName), destinationDir)
+	if err != nil {
+		return err
+	}
+
+	return filesystem.CopyPath(filepath.Join(srcDir, boltdb.DatabaseFileName), destinationDir)
 }

api/build/variables.go

@@ -0,0 +1,9 @@
+package build
+
+// Variables to be set during the build time
+var BuildNumber string
+var ImageTag string
+var NodejsVersion string
+var YarnVersion string
+var WebpackVersion string
+var GoVersion string

api/chisel/schedules.go

@@ -1,14 +1,13 @@
 package chisel
 
 import (
-	"strconv"
-
 	portainer "github.com/portainer/portainer/api"
 )
 
 // AddEdgeJob register an EdgeJob inside the tunnel details associated to an environment(endpoint).
 func (service *Service) AddEdgeJob(endpointID portainer.EndpointID, edgeJob *portainer.EdgeJob) {
-	tunnel := service.GetTunnelDetails(endpointID)
+	service.mu.Lock()
+	tunnel := service.getTunnelDetails(endpointID)
 
 	existingJobIndex := -1
 	for idx, existingJob := range tunnel.Jobs {
@@ -24,24 +23,25 @@ func (service *Service) AddEdgeJob(endpointID portainer.EndpointID, edgeJob *por
 		tunnel.Jobs[existingJobIndex] = *edgeJob
 	}
 
-	key := strconv.Itoa(int(endpointID))
-	service.tunnelDetailsMap.Set(key, tunnel)
+	service.mu.Unlock()
 }
 
 // RemoveEdgeJob will remove the specified Edge job from each tunnel it was registered with.
 func (service *Service) RemoveEdgeJob(edgeJobID portainer.EdgeJobID) {
-	for item := range service.tunnelDetailsMap.IterBuffered() {
-		tunnelDetails := item.Val.(*portainer.TunnelDetails)
+	service.mu.Lock()
 
-		updatedJobs := make([]portainer.EdgeJob, 0)
-		for _, edgeJob := range tunnelDetails.Jobs {
-			if edgeJob.ID == edgeJobID {
-				continue
+	for _, tunnel := range service.tunnelDetailsMap {
+		// Filter in-place
+		n := 0
+		for _, edgeJob := range tunnel.Jobs {
+			if edgeJob.ID != edgeJobID {
+				tunnel.Jobs[n] = edgeJob
+				n++
 			}
-			updatedJobs = append(updatedJobs, edgeJob)
 		}
 
-		tunnelDetails.Jobs = updatedJobs
-		service.tunnelDetailsMap.Set(item.Key, tunnelDetails)
+		tunnel.Jobs = tunnel.Jobs[:n]
 	}
+
+	service.mu.Unlock()
 }

api/chisel/service.go

@@ -3,17 +3,17 @@ package chisel
 import (
 	"context"
 	"fmt"
-	"github.com/portainer/portainer/api/http/proxy"
-	"log"
 	"net/http"
-	"strconv"
+	"sync"
 	"time"
 
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices"
+	"github.com/portainer/portainer/api/http/proxy"
+
 	"github.com/dchest/uniuri"
 	chserver "github.com/jpillora/chisel/server"
-	cmap "github.com/orcaman/concurrent-map"
-	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices"
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -28,18 +28,19 @@ const (
 type Service struct {
 	serverFingerprint string
 	serverPort        string
-	tunnelDetailsMap  cmap.ConcurrentMap
+	tunnelDetailsMap  map[portainer.EndpointID]*portainer.TunnelDetails
 	dataStore         dataservices.DataStore
 	snapshotService   portainer.SnapshotService
 	chiselServer      *chserver.Server
 	shutdownCtx       context.Context
 	ProxyManager      *proxy.Manager
+	mu                sync.Mutex
 }
 
 // NewService returns a pointer to a new instance of Service
 func NewService(dataStore dataservices.DataStore, shutdownCtx context.Context) *Service {
 	return &Service{
-		tunnelDetailsMap: cmap.New(),
+		tunnelDetailsMap: make(map[portainer.EndpointID]*portainer.TunnelDetails),
 		dataStore:        dataStore,
 		shutdownCtx:      shutdownCtx,
 	}
@@ -58,17 +59,17 @@ func (service *Service) pingAgent(endpointID portainer.EndpointID) error {
 		Timeout: 3 * time.Second,
 	}
 	_, err = httpClient.Do(req)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return err
 }
 
 // KeepTunnelAlive keeps the tunnel of the given environment for maxAlive duration, or until ctx is done
 func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx context.Context, maxAlive time.Duration) {
 	go func() {
-		log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: start for %.0f minutes]\n", endpointID, maxAlive.Minutes())
+		log.Debug().
+			Int("endpoint_id", int(endpointID)).
+			Float64("max_alive_minutes", maxAlive.Minutes()).
+			Msg("start")
+
 		maxAliveTicker := time.NewTicker(maxAlive)
 		defer maxAliveTicker.Stop()
 		pingTicker := time.NewTicker(tunnelCleanupInterval)
@@ -80,14 +81,25 @@ func (service *Service) KeepTunnelAlive(endpointID portainer.EndpointID, ctx con
 				service.SetTunnelStatusToActive(endpointID)
 				err := service.pingAgent(endpointID)
 				if err != nil {
-					log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [warning: ping agent err=%s]\n", endpointID, err)
+					log.Debug().
+						Int("endpoint_id", int(endpointID)).
+						Err(err).
+						Msg("ping agent")
 				}
 			case <-maxAliveTicker.C:
-				log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: stop as %.0f minutes timeout]\n", endpointID, maxAlive.Minutes())
+				log.Debug().
+					Int("endpoint_id", int(endpointID)).
+					Float64("timeout_minutes", maxAlive.Minutes()).
+					Msg("tunnel keep alive timeout")
+
 				return
 			case <-ctx.Done():
 				err := ctx.Err()
-				log.Printf("[DEBUG] [chisel,KeepTunnelAlive] [endpoint_id: %d] [message: stop as err=%s]\n", endpointID, err)
+				log.Debug().
+					Int("endpoint_id", int(endpointID)).
+					Err(err).
+					Msg("tunnel stop")
+
 				return
 			}
 		}
@@ -166,7 +178,10 @@ func (service *Service) retrievePrivateKeySeed() (string, error) {
 }
 
 func (service *Service) startTunnelVerificationLoop() {
-	log.Printf("[DEBUG] [chisel, monitoring] [check_interval_seconds: %f] [message: starting tunnel management process]", tunnelCleanupInterval.Seconds())
+	log.Debug().
+		Float64("check_interval_seconds", tunnelCleanupInterval.Seconds()).
+		Msg("starting tunnel management process")
+
 	ticker := time.NewTicker(tunnelCleanupInterval)
 
 	for {
@@ -174,10 +189,12 @@ func (service *Service) startTunnelVerificationLoop() {
 		case <-ticker.C:
 			service.checkTunnels()
 		case <-service.shutdownCtx.Done():
-			log.Println("[DEBUG] Shutting down tunnel service")
+			log.Debug().Msg("shutting down tunnel service")
+
 			if err := service.StopTunnelServer(); err != nil {
-				log.Printf("Stopped tunnel service: %s", err)
+				log.Debug().Err(err).Msg("stopped tunnel service")
 			}
+
 			ticker.Stop()
 			return
 		}
@@ -185,44 +202,57 @@ func (service *Service) startTunnelVerificationLoop() {
 }
 
 func (service *Service) checkTunnels() {
-	for item := range service.tunnelDetailsMap.IterBuffered() {
-		tunnel := item.Val.(*portainer.TunnelDetails)
+	tunnels := make(map[portainer.EndpointID]portainer.TunnelDetails)
 
+	service.mu.Lock()
+	for key, tunnel := range service.tunnelDetailsMap {
+		tunnels[key] = *tunnel
+	}
+	service.mu.Unlock()
+
+	for endpointID, tunnel := range tunnels {
 		if tunnel.LastActivity.IsZero() || tunnel.Status == portainer.EdgeAgentIdle {
 			continue
 		}
 
 		elapsed := time.Since(tunnel.LastActivity)
-		log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [message: environment tunnel monitoring]", item.Key, tunnel.Status, elapsed.Seconds())
+		log.Debug().
+			Int("endpoint_id", int(endpointID)).
+			Str("status", tunnel.Status).
+			Float64("status_time_seconds", elapsed.Seconds()).
+			Msg("environment tunnel monitoring")
 
 		if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed.Seconds() < requiredTimeout.Seconds() {
 			continue
 		} else if tunnel.Status == portainer.EdgeAgentManagementRequired && elapsed.Seconds() > requiredTimeout.Seconds() {
-			log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [timeout_seconds: %f] [message: REQUIRED state timeout exceeded]", item.Key, tunnel.Status, elapsed.Seconds(), requiredTimeout.Seconds())
+			log.Debug().
+				Int("endpoint_id", int(endpointID)).
+				Str("status", tunnel.Status).
+				Float64("status_time_seconds", elapsed.Seconds()).
+				Float64("timeout_seconds", requiredTimeout.Seconds()).
+				Msg("REQUIRED state timeout exceeded")
 		}
 
 		if tunnel.Status == portainer.EdgeAgentActive && elapsed.Seconds() < activeTimeout.Seconds() {
 			continue
 		} else if tunnel.Status == portainer.EdgeAgentActive && elapsed.Seconds() > activeTimeout.Seconds() {
-			log.Printf("[DEBUG] [chisel,monitoring] [endpoint_id: %s] [status: %s] [status_time_seconds: %f] [timeout_seconds: %f] [message: ACTIVE state timeout exceeded]", item.Key, tunnel.Status, elapsed.Seconds(), activeTimeout.Seconds())
+			log.Debug().
+				Int("endpoint_id", int(endpointID)).
+				Str("status", tunnel.Status).
+				Float64("status_time_seconds", elapsed.Seconds()).
+				Float64("timeout_seconds", activeTimeout.Seconds()).
+				Msg("ACTIVE state timeout exceeded")
 
-			endpointID, err := strconv.Atoi(item.Key)
+			err := service.snapshotEnvironment(endpointID, tunnel.Port)
 			if err != nil {
-				log.Printf("[ERROR] [chisel,snapshot,conversion] Invalid environment identifier (id: %s): %s", item.Key, err)
-			}
+				log.Error().
+					Int("endpoint_id", int(endpointID)).Err(
 
-			err = service.snapshotEnvironment(portainer.EndpointID(endpointID), tunnel.Port)
-			if err != nil {
-				log.Printf("[ERROR] [snapshot] Unable to snapshot Edge environment (id: %s): %s", item.Key, err)
+					err).
+					Msg("unable to snapshot Edge environment")
 			}
 		}
 
-		endpointID, err := strconv.Atoi(item.Key)
-		if err != nil {
-			log.Printf("[ERROR] [chisel,conversion] Invalid environment identifier (id: %s): %s", item.Key, err)
-			continue
-		}
-
 		service.SetTunnelStatusToIdle(portainer.EndpointID(endpointID))
 	}
 }

api/chisel/tunnel.go

@@ -4,13 +4,11 @@ import (
 	"encoding/base64"
 	"fmt"
 	"math/rand"
-	"strconv"
 	"strings"
 	"time"
 
-	"github.com/portainer/libcrypto"
-
 	"github.com/dchest/uniuri"
+	"github.com/portainer/libcrypto"
 	portainer "github.com/portainer/portainer/api"
 )
 
@@ -19,13 +17,13 @@ const (
 	maxAvailablePort = 65535
 )
 
+// NOTE: it needs to be called with the lock acquired
 // getUnusedPort is used to generate an unused random port in the dynamic port range.
 // Dynamic ports (also called private ports) are 49152 to 65535.
 func (service *Service) getUnusedPort() int {
 	port := randomInt(minAvailablePort, maxAvailablePort)
 
-	for item := range service.tunnelDetailsMap.IterBuffered() {
-		tunnel := item.Val.(*portainer.TunnelDetails)
+	for _, tunnel := range service.tunnelDetailsMap {
 		if tunnel.Port == port {
 			return service.getUnusedPort()
 		}
@@ -38,26 +36,32 @@ func randomInt(min, max int) int {
 	return min + rand.Intn(max-min)
 }
 
+// NOTE: it needs to be called with the lock acquired
+func (service *Service) getTunnelDetails(endpointID portainer.EndpointID) *portainer.TunnelDetails {
+
+	if tunnel, ok := service.tunnelDetailsMap[endpointID]; ok {
+		return tunnel
+	}
+
+	tunnel := &portainer.TunnelDetails{
+		Status: portainer.EdgeAgentIdle,
+	}
+
+	service.tunnelDetailsMap[endpointID] = tunnel
+
+	return tunnel
+}
+
 // GetTunnelDetails returns information about the tunnel associated to an environment(endpoint).
-func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) *portainer.TunnelDetails {
-	key := strconv.Itoa(int(endpointID))
+func (service *Service) GetTunnelDetails(endpointID portainer.EndpointID) portainer.TunnelDetails {
+	service.mu.Lock()
+	defer service.mu.Unlock()
 
-	if item, ok := service.tunnelDetailsMap.Get(key); ok {
-		tunnelDetails := item.(*portainer.TunnelDetails)
-		return tunnelDetails
-	}
-
-	jobs := make([]portainer.EdgeJob, 0)
-	return &portainer.TunnelDetails{
-		Status:      portainer.EdgeAgentIdle,
-		Port:        0,
-		Jobs:        jobs,
-		Credentials: "",
-	}
+	return *service.getTunnelDetails(endpointID)
 }
 
 // GetActiveTunnel retrieves an active tunnel which allows communicating with edge agent
-func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (*portainer.TunnelDetails, error) {
+func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (portainer.TunnelDetails, error) {
 	tunnel := service.GetTunnelDetails(endpoint.ID)
 
 	if tunnel.Status == portainer.EdgeAgentActive {
@@ -68,13 +72,13 @@ func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (*portaine
 	if tunnel.Status == portainer.EdgeAgentIdle || tunnel.Status == portainer.EdgeAgentManagementRequired {
 		err := service.SetTunnelStatusToRequired(endpoint.ID)
 		if err != nil {
-			return nil, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
+			return portainer.TunnelDetails{}, fmt.Errorf("failed opening tunnel to endpoint: %w", err)
 		}
 
 		if endpoint.EdgeCheckinInterval == 0 {
 			settings, err := service.dataStore.Settings().Settings()
 			if err != nil {
-				return nil, fmt.Errorf("failed fetching settings from db: %w", err)
+				return portainer.TunnelDetails{}, fmt.Errorf("failed fetching settings from db: %w", err)
 			}
 
 			endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
@@ -83,29 +87,27 @@ func (service *Service) GetActiveTunnel(endpoint *portainer.Endpoint) (*portaine
 		time.Sleep(2 * time.Duration(endpoint.EdgeCheckinInterval) * time.Second)
 	}
 
-	tunnel = service.GetTunnelDetails(endpoint.ID)
-
-	return tunnel, nil
+	return service.GetTunnelDetails(endpoint.ID), nil
 }
 
 // SetTunnelStatusToActive update the status of the tunnel associated to the specified environment(endpoint).
 // It sets the status to ACTIVE.
 func (service *Service) SetTunnelStatusToActive(endpointID portainer.EndpointID) {
-	tunnel := service.GetTunnelDetails(endpointID)
+	service.mu.Lock()
+	tunnel := service.getTunnelDetails(endpointID)
 	tunnel.Status = portainer.EdgeAgentActive
 	tunnel.Credentials = ""
 	tunnel.LastActivity = time.Now()
-
-	key := strconv.Itoa(int(endpointID))
-	service.tunnelDetailsMap.Set(key, tunnel)
+	service.mu.Unlock()
 }
 
 // SetTunnelStatusToIdle update the status of the tunnel associated to the specified environment(endpoint).
 // It sets the status to IDLE.
 // It removes any existing credentials associated to the tunnel.
 func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
-	tunnel := service.GetTunnelDetails(endpointID)
+	service.mu.Lock()
 
+	tunnel := service.getTunnelDetails(endpointID)
 	tunnel.Status = portainer.EdgeAgentIdle
 	tunnel.Port = 0
 	tunnel.LastActivity = time.Now()
@@ -116,10 +118,9 @@ func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
 		service.chiselServer.DeleteUser(strings.Split(credentials, ":")[0])
 	}
 
-	key := strconv.Itoa(int(endpointID))
-	service.tunnelDetailsMap.Set(key, tunnel)
-
 	service.ProxyManager.DeleteEndpointProxy(endpointID)
+
+	service.mu.Unlock()
 }
 
 // SetTunnelStatusToRequired update the status of the tunnel associated to the specified environment(endpoint).
@@ -128,7 +129,10 @@ func (service *Service) SetTunnelStatusToIdle(endpointID portainer.EndpointID) {
 // and generate temporary credentials that can be used to establish a reverse tunnel on that port.
 // Credentials are encrypted using the Edge ID associated to the environment(endpoint).
 func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointID) error {
-	tunnel := service.GetTunnelDetails(endpointID)
+	tunnel := service.getTunnelDetails(endpointID)
+
+	service.mu.Lock()
+	defer service.mu.Unlock()
 
 	if tunnel.Port == 0 {
 		endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
@@ -152,9 +156,6 @@ func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointI
 			return err
 		}
 		tunnel.Credentials = credentials
-
-		key := strconv.Itoa(int(endpointID))
-		service.tunnelDetailsMap.Set(key, tunnel)
 	}
 
 	return nil

api/cli/cli.go

@@ -2,15 +2,14 @@ package cli
 
 import (
 	"errors"
-	"log"
+	"os"
+	"path/filepath"
+	"strings"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
 
-	"os"
-	"path/filepath"
-	"strings"
-
+	"github.com/rs/zerolog/log"
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
@@ -35,6 +34,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		TunnelPort:                kingpin.Flag("tunnel-port", "Port to serve the tunnel server").Default(defaultTunnelServerPort).String(),
 		Assets:                    kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
 		Data:                      kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
+		DemoEnvironment:           kingpin.Flag("demo", "Demo environment").Bool(),
 		EndpointURL:               kingpin.Flag("host", "Environment URL").Short('H').String(),
 		FeatureFlags:              BoolPairs(kingpin.Flag("feat", "List of feature flags").Hidden()),
 		EnableEdgeComputeFeatures: kingpin.Flag("edge-compute", "Enable Edge Compute features").Bool(),
@@ -50,13 +50,19 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		SSLCert:                   kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(),
 		SSLKey:                    kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(),
 		Rollback:                  kingpin.Flag("rollback", "Rollback the database store to the previous version").Bool(),
-		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").Default(defaultSnapshotInterval).String(),
-		AdminPassword:             kingpin.Flag("admin-password", "Hashed admin password").String(),
+		SnapshotInterval:          kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").String(),
+		AdminPassword:             kingpin.Flag("admin-password", "Set admin password with provided hash").String(),
 		AdminPasswordFile:         kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
 		Labels:                    pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')),
 		Logo:                      kingpin.Flag("logo", "URL for the logo displayed in the UI").String(),
 		Templates:                 kingpin.Flag("templates", "URL to the templates definitions.").Short('t').String(),
 		BaseURL:                   kingpin.Flag("base-url", "Base URL parameter such as portainer if running portainer as http://yourdomain.com/portainer/.").Short('b').Default(defaultBaseURL).String(),
+		InitialMmapSize:           kingpin.Flag("initial-mmap-size", "Initial mmap size of the database in bytes").Int(),
+		MaxBatchSize:              kingpin.Flag("max-batch-size", "Maximum size of a batch").Int(),
+		MaxBatchDelay:             kingpin.Flag("max-batch-delay", "Maximum delay before a batch starts").Duration(),
+		SecretKeyName:             kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/<secret-key-name>.").Default(defaultSecretKeyName).String(),
+		LogLevel:                  kingpin.Flag("log-level", "Set the minimum logging level to show").Default("INFO").Enum("DEBUG", "INFO", "WARN", "ERROR"),
+		LogMode:                   kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("PRETTY", "JSON"),
 	}
 
 	kingpin.Parse()
@@ -96,11 +102,11 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 
 func displayDeprecationWarnings(flags *portainer.CLIFlags) {
 	if *flags.NoAnalytics {
-		log.Println("Warning: The --no-analytics flag has been kept to allow migration of instances running a previous version of Portainer with this flag enabled, to version 2.0 where enabling this flag will have no effect.")
+		log.Warn().Msg("the --no-analytics flag has been kept to allow migration of instances running a previous version of Portainer with this flag enabled, to version 2.0 where enabling this flag will have no effect")
 	}
 
 	if *flags.SSL {
-		log.Println("Warning: SSL is enabled by default and there is no need for the --ssl flag. It has been kept to allow migration of instances running a previous version of Portainer with this flag enabled")
+		log.Warn().Msg("SSL is enabled by default and there is no need for the --ssl flag, it has been kept to allow migration of instances running a previous version of Portainer with this flag enabled")
 	}
 }
 
@@ -125,7 +131,7 @@ func validateEndpointURL(endpointURL string) error {
 }
 
 func validateSnapshotInterval(snapshotInterval string) error {
-	if snapshotInterval != defaultSnapshotInterval {
+	if snapshotInterval != "" {
 		_, err := time.ParseDuration(snapshotInterval)
 		if err != nil {
 			return errInvalidSnapshotInterval

api/cli/defaults.go

@@ -18,8 +18,6 @@ const (
 	defaultHTTPDisabled        = "false"
 	defaultHTTPEnabled         = "false"
 	defaultSSL                 = "false"
-	defaultSSLCertPath         = "/certs/portainer.crt"
-	defaultSSLKeyPath          = "/certs/portainer.key"
-	defaultSnapshotInterval    = "5m"
 	defaultBaseURL             = "/"
+	defaultSecretKeyName       = "portainer"
 )

api/cli/defaults_windows.go

@@ -15,8 +15,7 @@ const (
 	defaultHTTPDisabled        = "false"
 	defaultHTTPEnabled         = "false"
 	defaultSSL                 = "false"
-	defaultSSLCertPath         = "C:\\certs\\portainer.crt"
-	defaultSSLKeyPath          = "C:\\certs\\portainer.key"
 	defaultSnapshotInterval    = "5m"
 	defaultBaseURL             = "/"
+	defaultSecretKeyName       = "portainer"
 )

api/cmd/portainer/import.go

@@ -1,11 +1,10 @@
 package main
 
 import (
-	"log"
-
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/datastore"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 func importFromJson(fileService portainer.FileService, store *datastore.Store) {
@@ -13,17 +12,17 @@ func importFromJson(fileService portainer.FileService, store *datastore.Store) {
 	importFile := "/data/import.json"
 	if exists, _ := fileService.FileExists(importFile); exists {
 		if err := store.Import(importFile); err != nil {
-			logrus.WithError(err).Debugf("import %s failed", importFile)
-
+			log.Error().Str("filename", importFile).Err(err).Msg("import failed")
 			// TODO: should really rollback on failure, but then we have nothing.
 		} else {
-			logrus.Printf("Successfully imported %s to new portainer database", importFile)
+			log.Info().Str("filename", importFile).Msg("successfully imported the file to a new portainer database")
 		}
+
 		// TODO: this is bad - its to ensure that any defaults that were broken in import, or migrations get set back to what we want
 		// I also suspect that everything from "Init to Init" is potentially a migration
 		err := store.Init()
 		if err != nil {
-			log.Fatalf("failed initializing data store: %v", err)
+			log.Fatal().Err(err).Msg("failed initializing data store")
 		}
 	}
 }

api/cmd/portainer/log.go

@@ -1,19 +1,55 @@
 package main
 
 import (
-	"log"
+	"fmt"
+	stdlog "log"
+	"os"
 
-	"github.com/sirupsen/logrus"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/rs/zerolog/pkgerrors"
 )
 
 func configureLogger() {
-	logger := logrus.New() // logger is to implicitly substitute stdlib's log
-	log.SetOutput(logger.Writer())
+	zerolog.ErrorStackFieldName = "stack_trace"
+	zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack
+	zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
 
-	formatter := &logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true}
-	logger.SetFormatter(formatter)
-	logrus.SetFormatter(formatter)
+	stdlog.SetFlags(0)
+	stdlog.SetOutput(log.Logger)
 
-	logger.SetLevel(logrus.DebugLevel)
-	logrus.SetLevel(logrus.DebugLevel)
+	log.Logger = log.Logger.With().Caller().Stack().Logger()
+}
+
+func setLoggingLevel(level string) {
+	switch level {
+	case "ERROR":
+		zerolog.SetGlobalLevel(zerolog.ErrorLevel)
+	case "WARN":
+		zerolog.SetGlobalLevel(zerolog.WarnLevel)
+	case "INFO":
+		zerolog.SetGlobalLevel(zerolog.InfoLevel)
+	case "DEBUG":
+		zerolog.SetGlobalLevel(zerolog.DebugLevel)
+	}
+}
+
+func setLoggingMode(mode string) {
+	switch mode {
+	case "PRETTY":
+		log.Logger = log.Output(zerolog.ConsoleWriter{
+			Out:           os.Stderr,
+			NoColor:       true,
+			TimeFormat:    "2006/01/02 03:04PM",
+			FormatMessage: formatMessage})
+	case "JSON":
+		log.Logger = log.Output(os.Stderr)
+	}
+}
+
+func formatMessage(i interface{}) string {
+	if i == nil {
+		return ""
+	}
+	return fmt.Sprintf("%s |", i)
 }

api/cmd/portainer/main.go

@@ -2,25 +2,26 @@ package main
 
 import (
 	"context"
+	"crypto/sha256"
 	"fmt"
-	"log"
 	"os"
 	"path"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/sirupsen/logrus"
-
 	"github.com/portainer/libhelm"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/apikey"
+	"github.com/portainer/portainer/api/build"
 	"github.com/portainer/portainer/api/chisel"
 	"github.com/portainer/portainer/api/cli"
 	"github.com/portainer/portainer/api/crypto"
 	"github.com/portainer/portainer/api/database"
+	"github.com/portainer/portainer/api/database/boltdb"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/demo"
 	"github.com/portainer/portainer/api/docker"
 	"github.com/portainer/portainer/api/exec"
 	"github.com/portainer/portainer/api/filesystem"
@@ -41,81 +42,97 @@ import (
 	"github.com/portainer/portainer/api/oauth"
 	"github.com/portainer/portainer/api/scheduler"
 	"github.com/portainer/portainer/api/stacks"
+
+	"github.com/rs/zerolog/log"
 )
 
 func initCLI() *portainer.CLIFlags {
 	var cliService portainer.CLIService = &cli.Service{}
 	flags, err := cliService.ParseFlags(portainer.APIVersion)
 	if err != nil {
-		log.Fatalf("failed parsing flags: %v", err)
+		log.Fatal().Err(err).Msg("failed parsing flags")
 	}
 
 	err = cliService.ValidateFlags(flags)
 	if err != nil {
-		log.Fatalf("failed validating flags:%v", err)
+		log.Fatal().Err(err).Msg("failed validating flags")
 	}
+
 	return flags
 }
 
 func initFileService(dataStorePath string) portainer.FileService {
 	fileService, err := filesystem.NewService(dataStorePath, "")
 	if err != nil {
-		log.Fatalf("failed creating file service: %v", err)
+		log.Fatal().Err(err).Msg("failed creating file service")
 	}
+
 	return fileService
 }
 
-func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore {
-	connection, err := database.NewDatabase("boltdb", *flags.Data)
+func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore {
+	connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey)
 	if err != nil {
-		panic(err)
+		log.Fatal().Err(err).Msg("failed creating database connection")
 	}
+
+	if bconn, ok := connection.(*boltdb.DbConnection); ok {
+		bconn.MaxBatchSize = *flags.MaxBatchSize
+		bconn.MaxBatchDelay = *flags.MaxBatchDelay
+		bconn.InitialMmapSize = *flags.InitialMmapSize
+	} else {
+		log.Fatal().Msg("failed creating database connection: expecting a boltdb database type but a different one was received")
+	}
+
 	store := datastore.NewStore(*flags.Data, fileService, connection)
 	isNew, err := store.Open()
 	if err != nil {
-		log.Fatalf("failed opening store: %v", err)
+		log.Fatal().Err(err).Msg("failed opening store")
 	}
 
 	if *flags.Rollback {
 		err := store.Rollback(false)
 		if err != nil {
-			log.Fatalf("failed rolling back: %s", err)
+			log.Fatal().Err(err).Msg("failed rolling back")
 		}
 
-		log.Println("Exiting rollback")
+		log.Info().Msg("exiting rollback")
 		os.Exit(0)
+
 		return nil
 	}
 
-	// Init sets some defaults - its basically a migration
+	// Init sets some defaults - it's basically a migration
 	err = store.Init()
 	if err != nil {
-		log.Fatalf("failed initializing data store: %v", err)
+		log.Fatal().Err(err).Msg("failed initializing data store")
 	}
 
 	if isNew {
 		// from MigrateData
 		store.VersionService.StoreDBVersion(portainer.DBVersion)
 
-		// Disabled for now.  Can't use feature flags due to the way that works
-		// EXPERIMENTAL, will only activate if `/data/import.json` exists
-		//importFromJson(fileService, store)
-
 		err := updateSettingsFromFlags(store, flags)
 		if err != nil {
-			log.Fatalf("failed updating settings from flags: %v", err)
+			log.Fatal().Err(err).Msg("failed updating settings from flags")
+		}
+	} else {
+		storedVersion, err := store.VersionService.DBVersion()
+		if err != nil {
+			log.Fatal().Err(err).Msg("failure during creation of new database")
+		}
+
+		if storedVersion != portainer.DBVersion {
+			err = store.MigrateData()
+			if err != nil {
+				log.Fatal().Err(err).Msg("failed migration")
+			}
 		}
 	}
 
-	storedVersion, err := store.VersionService.DBVersion()
+	err = updateSettingsFromFlags(store, flags)
 	if err != nil {
-		log.Fatalf("Something failed during creation of new database: %v", err)
-	}
-	if storedVersion != portainer.DBVersion {
-		err = store.MigrateData()
-		if err != nil {
-			log.Fatalf("failed migration: %v", err)
-		}
+		log.Fatal().Err(err).Msg("failed updating settings from flags")
 	}
 
 	// this is for the db restore functionality - needs more tests.
@@ -127,19 +144,19 @@ func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService,
 
 		err := store.Export(exportFilename)
 		if err != nil {
-			logrus.WithError(err).Debugf("failed to export to %s", exportFilename)
+			log.Error().Str("filename", exportFilename).Err(err).Msg("failed to export")
 		} else {
-			logrus.Debugf("exported to %s", exportFilename)
+			log.Debug().Str("filename", exportFilename).Msg("exported")
 		}
-		connection.Close()
 	}()
+
 	return store
 }
 
 func initComposeStackManager(assetsPath string, configPath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
 	composeWrapper, err := exec.NewComposeStackManager(assetsPath, configPath, proxyManager)
 	if err != nil {
-		log.Fatalf("failed creating compose manager: %s", err)
+		log.Fatal().Err(err).Msg("failed creating compose manager")
 	}
 
 	return composeWrapper
@@ -173,6 +190,7 @@ func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore)
 	if err != nil {
 		return nil, err
 	}
+
 	return jwtService, nil
 }
 
@@ -192,11 +210,11 @@ func initOAuthService() portainer.OAuthService {
 	return oauth.NewService()
 }
 
-func initGitService() portainer.GitService {
-	return git.NewService()
+func initGitService(ctx context.Context) portainer.GitService {
+	return git.NewService(ctx)
 }
 
-func initSSLService(addr, dataPath, certPath, keyPath string, fileService portainer.FileService, dataStore dataservices.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
+func initSSLService(addr, certPath, keyPath string, fileService portainer.FileService, dataStore dataservices.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
 	slices := strings.Split(addr, ":")
 	host := slices[0]
 	if host == "" {
@@ -217,15 +235,15 @@ func initDockerClientFactory(signatureService portainer.DigitalSignatureService,
 	return docker.NewClientFactory(signatureService, reverseTunnelService)
 }
 
-func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, instanceID string, dataStore dataservices.DataStore) *kubecli.ClientFactory {
-	return kubecli.NewClientFactory(signatureService, reverseTunnelService, instanceID, dataStore)
+func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, dataStore dataservices.DataStore, instanceID, addrHTTPS, userSessionTimeout string) (*kubecli.ClientFactory, error) {
+	return kubecli.NewClientFactory(signatureService, reverseTunnelService, dataStore, instanceID, addrHTTPS, userSessionTimeout)
 }
 
-func initSnapshotService(snapshotInterval string, dataStore dataservices.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory, shutdownCtx context.Context) (portainer.SnapshotService, error) {
+func initSnapshotService(snapshotIntervalFromFlag string, dataStore dataservices.DataStore, dockerClientFactory *docker.ClientFactory, kubernetesClientFactory *kubecli.ClientFactory, shutdownCtx context.Context) (portainer.SnapshotService, error) {
 	dockerSnapshotter := docker.NewSnapshotter(dockerClientFactory)
 	kubernetesSnapshotter := kubernetes.NewSnapshotter(kubernetesClientFactory)
 
-	snapshotService, err := snapshot.NewService(snapshotInterval, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx)
+	snapshotService, err := snapshot.NewService(snapshotIntervalFromFlag, dataStore, dockerSnapshotter, kubernetesSnapshotter, shutdownCtx)
 	if err != nil {
 		return nil, err
 	}
@@ -246,11 +264,17 @@ func updateSettingsFromFlags(dataStore dataservices.DataStore, flags *portainer.
 		return err
 	}
 
-	settings.LogoURL = *flags.Logo
-	settings.SnapshotInterval = *flags.SnapshotInterval
-	settings.EnableEdgeComputeFeatures = *flags.EnableEdgeComputeFeatures
-	settings.EnableTelemetry = true
-	settings.OAuthSettings.SSO = true
+	if *flags.SnapshotInterval != "" {
+		settings.SnapshotInterval = *flags.SnapshotInterval
+	}
+
+	if *flags.Logo != "" {
+		settings.LogoURL = *flags.Logo
+	}
+
+	if *flags.EnableEdgeComputeFeatures {
+		settings.EnableEdgeComputeFeatures = *flags.EnableEdgeComputeFeatures
+	}
 
 	if *flags.Templates != "" {
 		settings.TemplatesURL = *flags.Templates
@@ -260,6 +284,12 @@ func updateSettingsFromFlags(dataStore dataservices.DataStore, flags *portainer.
 		settings.BlackListedLabels = *flags.Labels
 	}
 
+	if agentKey, ok := os.LookupEnv("AGENT_SECRET"); ok {
+		settings.AgentSecret = agentKey
+	} else {
+		settings.AgentSecret = ""
+	}
+
 	err = dataStore.Settings().UpdateSettings(settings)
 	if err != nil {
 		return err
@@ -272,8 +302,8 @@ func updateSettingsFromFlags(dataStore dataservices.DataStore, flags *portainer.
 
 	if *flags.HTTPDisabled {
 		sslSettings.HTTPEnabled = false
-	} else {
-		sslSettings.HTTPEnabled = *flags.HTTPEnabled || sslSettings.HTTPEnabled
+	} else if *flags.HTTPEnabled {
+		sslSettings.HTTPEnabled = true
 	}
 
 	err = dataStore.SSLSettings().UpdateSettings(sslSettings)
@@ -315,11 +345,7 @@ func enableFeaturesFromFlags(dataStore dataservices.DataStore, flags *portainer.
 			return fmt.Errorf("feature flag's '%s' value should be true or false", feat.Name)
 		}
 
-		if featureState {
-			log.Printf("Feature %v : on", *correspondingFeature)
-		} else {
-			log.Printf("Feature %v : off", *correspondingFeature)
-		}
+		log.Info().Str("feature", string(*correspondingFeature)).Bool("state", featureState).Msg("")
 
 		settings.FeatureFlagSettings[*correspondingFeature] = featureState
 	}
@@ -347,7 +373,7 @@ func generateAndStoreKeyPair(fileService portainer.FileService, signatureService
 func initKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
 	existingKeyPair, err := fileService.KeyPairFilesExist()
 	if err != nil {
-		log.Fatalf("failed checking for existing key pair: %v", err)
+		log.Fatal().Err(err).Msg("failed checking for existing key pair")
 	}
 
 	if existingKeyPair {
@@ -380,7 +406,6 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore dataservices.
 		TLSConfig:          tlsConfiguration,
 		UserAccessPolicies: portainer.UserAccessPolicies{},
 		TeamAccessPolicies: portainer.TeamAccessPolicies{},
-		Extensions:         []portainer.EndpointExtension{},
 		TagIDs:             []portainer.TagID{},
 		Status:             portainer.EndpointStatusUp,
 		Snapshots:          []portainer.DockerSnapshot{},
@@ -418,7 +443,11 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore dataservices.
 
 	err := snapshotService.SnapshotEndpoint(endpoint)
 	if err != nil {
-		log.Printf("http error: environment snapshot error (environment=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+		log.Error().
+			Str("endpoint", endpoint.Name).
+			Str("URL", endpoint.URL).
+			Err(err).
+			Msg("environment snapshot error")
 	}
 
 	return dataStore.Endpoint().Create(endpoint)
@@ -442,7 +471,6 @@ func createUnsecuredEndpoint(endpointURL string, dataStore dataservices.DataStor
 		TLSConfig:          portainer.TLSConfiguration{},
 		UserAccessPolicies: portainer.UserAccessPolicies{},
 		TeamAccessPolicies: portainer.TeamAccessPolicies{},
-		Extensions:         []portainer.EndpointExtension{},
 		TagIDs:             []portainer.TagID{},
 		Status:             portainer.EndpointStatusUp,
 		Snapshots:          []portainer.DockerSnapshot{},
@@ -464,7 +492,10 @@ func createUnsecuredEndpoint(endpointURL string, dataStore dataservices.DataStor
 
 	err := snapshotService.SnapshotEndpoint(endpoint)
 	if err != nil {
-		log.Printf("http error: environment snapshot error (environment=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+		log.Error().
+			Str("endpoint", endpoint.Name).
+			Str("URL", endpoint.URL).Err(err).
+			Msg("environment snapshot error")
 	}
 
 	return dataStore.Endpoint().Create(endpoint)
@@ -481,7 +512,8 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, s
 	}
 
 	if len(endpoints) > 0 {
-		log.Println("Instance already has defined environments. Skipping the environment defined via CLI.")
+		log.Info().Msg("instance already has defined environments, skipping the environment defined via CLI")
+
 		return nil
 	}
 
@@ -491,69 +523,94 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, s
 	return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotService)
 }
 
+func loadEncryptionSecretKey(keyfilename string) []byte {
+	content, err := os.ReadFile(path.Join("/run/secrets", keyfilename))
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Info().Str("filename", keyfilename).Msg("encryption key file not present")
+		} else {
+			log.Info().Err(err).Msg("error reading encryption key file")
+		}
+
+		return nil
+	}
+
+	// return a 32 byte hash of the secret (required for AES)
+	hash := sha256.Sum256(content)
+	return hash[:]
+}
+
 func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	shutdownCtx, shutdownTrigger := context.WithCancel(context.Background())
 
 	fileService := initFileService(*flags.Data)
+	encryptionKey := loadEncryptionSecretKey(*flags.SecretKeyName)
+	if encryptionKey == nil {
+		log.Info().Msg("proceeding without encryption key")
+	}
 
-	dataStore := initDataStore(flags, fileService, shutdownCtx)
+	dataStore := initDataStore(flags, encryptionKey, fileService, shutdownCtx)
 
 	if err := dataStore.CheckCurrentEdition(); err != nil {
-		log.Fatal(err)
+		log.Fatal().Err(err).Msg("")
 	}
+
 	instanceID, err := dataStore.Version().InstanceID()
 	if err != nil {
-		log.Fatalf("failed getting instance id: %v", err)
+		log.Fatal().Err(err).Msg("failed getting instance id")
 	}
 
 	apiKeyService := initAPIKeyService(dataStore)
 
 	settings, err := dataStore.Settings().Settings()
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal().Err(err).Msg("")
 	}
+
 	jwtService, err := initJWTService(settings.UserSessionTimeout, dataStore)
 	if err != nil {
-		log.Fatalf("failed initializing JWT service: %v", err)
+		log.Fatal().Err(err).Msg("failed initializing JWT service")
 	}
 
 	err = enableFeaturesFromFlags(dataStore, flags)
 	if err != nil {
-		log.Fatalf("failed enabling feature flag: %v", err)
+		log.Fatal().Err(err).Msg("failed enabling feature flag")
 	}
 
 	ldapService := initLDAPService()
-	oauthService := initOAuthService()
-	gitService := initGitService()
 
-	openAMTService := openamt.NewService(dataStore)
+	oauthService := initOAuthService()
+	gitService := initGitService(shutdownCtx)
+
+	openAMTService := openamt.NewService()
 
 	cryptoService := initCryptoService()
+
 	digitalSignatureService := initDigitalSignatureService()
 
-	sslService, err := initSSLService(*flags.AddrHTTPS, *flags.Data, *flags.SSLCert, *flags.SSLKey, fileService, dataStore, shutdownTrigger)
+	sslService, err := initSSLService(*flags.AddrHTTPS, *flags.SSLCert, *flags.SSLKey, fileService, dataStore, shutdownTrigger)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal().Err(err).Msg("")
 	}
 
 	sslSettings, err := sslService.GetSSLSettings()
 	if err != nil {
-		log.Fatalf("failed to get ssl settings: %s", err)
+		log.Fatal().Err(err).Msg("failed to get SSL settings")
 	}
 
 	err = initKeyPair(fileService, digitalSignatureService)
 	if err != nil {
-		log.Fatalf("failed initializing key pair: %v", err)
+		log.Fatal().Err(err).Msg("failed initializing key pair")
 	}
 
 	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
 
 	dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
-	kubernetesClientFactory := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, instanceID, dataStore)
+	kubernetesClientFactory, err := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, dataStore, instanceID, *flags.AddrHTTPS, settings.UserSessionTimeout)
 
 	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
 	if err != nil {
-		log.Fatalf("failed initializing snapshot service: %v", err)
+		log.Fatal().Err(err).Msg("failed initializing snapshot service")
 	}
 	snapshotService.Start()
 
@@ -562,9 +619,9 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	kubernetesTokenCacheManager := kubeproxy.NewTokenCacheManager()
 
-	kubeConfigService := kubernetes.NewKubeConfigCAService(*flags.AddrHTTPS, sslSettings.CertPath)
+	kubeClusterAccessService := kubernetes.NewKubeClusterAccessService(*flags.BaseURL, *flags.AddrHTTPS, sslSettings.CertPath)
 
-	proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager)
+	proxyManager := proxy.NewManager(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService)
 
 	reverseTunnelService.ProxyManager = proxyManager
 
@@ -574,37 +631,46 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore)
 	if err != nil {
-		log.Fatalf("failed initializing swarm stack manager: %s", err)
+		log.Fatal().Err(err).Msg("failed initializing swarm stack manager")
 	}
 
 	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
 
 	helmPackageManager, err := initHelmPackageManager(*flags.Assets)
 	if err != nil {
-		log.Fatalf("failed initializing helm package manager: %s", err)
+		log.Fatal().Err(err).Msg("failed initializing helm package manager")
 	}
 
 	err = edge.LoadEdgeJobs(dataStore, reverseTunnelService)
 	if err != nil {
-		log.Fatalf("failed loading edge jobs from database: %v", err)
+		log.Fatal().Err(err).Msg("failed loading edge jobs from database")
 	}
 
 	applicationStatus := initStatus(instanceID)
 
+	demoService := demo.NewService()
+	if *flags.DemoEnvironment {
+		err := demoService.Init(dataStore, cryptoService)
+		if err != nil {
+			log.Fatal().Err(err).Msg("failed initializing demo environment")
+		}
+	}
+
 	err = initEndpoint(flags, dataStore, snapshotService)
 	if err != nil {
-		log.Fatalf("failed initializing environment: %v", err)
+		log.Fatal().Err(err).Msg("failed initializing environment")
 	}
 
 	adminPasswordHash := ""
 	if *flags.AdminPasswordFile != "" {
 		content, err := fileService.GetFileContent(*flags.AdminPasswordFile, "")
 		if err != nil {
-			log.Fatalf("failed getting admin password file: %v", err)
+			log.Fatal().Err(err).Msg("failed getting admin password file")
 		}
+
 		adminPasswordHash, err = cryptoService.Hash(strings.TrimSuffix(string(content), "\n"))
 		if err != nil {
-			log.Fatalf("failed hashing admin password: %v", err)
+			log.Fatal().Err(err).Msg("failed hashing admin password")
 		}
 	} else if *flags.AdminPassword != "" {
 		adminPasswordHash = *flags.AdminPassword
@@ -613,39 +679,57 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	if adminPasswordHash != "" {
 		users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
 		if err != nil {
-			log.Fatalf("failed getting admin user: %v", err)
+			log.Fatal().Err(err).Msg("failed getting admin user")
 		}
 
 		if len(users) == 0 {
-			log.Println("Created admin user with the given password.")
+			log.Info().Msg("created admin user with the given password.")
 			user := &portainer.User{
 				Username: "admin",
 				Role:     portainer.AdministratorRole,
 				Password: adminPasswordHash,
 			}
+
 			err := dataStore.User().Create(user)
 			if err != nil {
-				log.Fatalf("failed creating admin user: %v", err)
+				log.Fatal().Err(err).Msg("failed creating admin user")
 			}
 		} else {
-			log.Println("Instance already has an administrator user defined. Skipping admin password related flags.")
+			log.Info().Msg("instance already has an administrator user defined, skipping admin password related flags.")
 		}
 	}
 
 	err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
 	if err != nil {
-		log.Fatalf("failed starting tunnel server: %s", err)
+		log.Fatal().Err(err).Msg("failed starting tunnel server")
 	}
 
 	sslDBSettings, err := dataStore.SSLSettings().Settings()
 	if err != nil {
-		log.Fatalf("failed to fetch ssl settings from DB")
+		log.Fatal().Msg("failed to fetch SSL settings from DB")
 	}
 
 	scheduler := scheduler.NewScheduler(shutdownCtx)
 	stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager, kubernetesDeployer)
 	stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService)
 
+	// FIXME: In 2.16 we changed the way ingress controller permissions are
+	// stored. Instead of being stored as annotation on an ingress rule, we keep
+	// them in our database. However, in order to run the migration we need an
+	// admin kube client to run lookup the old ingress rules and compare them
+	// with the current existing ingress classes.
+	//
+	// Unfortunately, our migrations run as part of the database initialization
+	// and our kubeclients require an initialized database. So it is not
+	// possible to do this migration as part of our normal flow. We DO have a
+	// migration which toggles a boolean in kubernetes configuration that
+	// indicated that this "post init" migration should be run. If/when this is
+	// resolved we can remove this function.
+	err = kubernetesClientFactory.PostInitMigrateIngresses()
+	if err != nil {
+		log.Fatal().Err(err).Msg("failure during creation of new database")
+	}
+
 	return &http.Server{
 		AuthorizationService:        authorizationService,
 		ReverseTunnelService:        reverseTunnelService,
@@ -669,7 +753,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		OpenAMTService:              openAMTService,
 		ProxyManager:                proxyManager,
 		KubernetesTokenCacheManager: kubernetesTokenCacheManager,
-		KubeConfigService:           kubeConfigService,
+		KubeClusterAccessService:    kubeClusterAccessService,
 		SignatureService:            digitalSignatureService,
 		SnapshotService:             snapshotService,
 		SSLService:                  sslService,
@@ -679,19 +763,32 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 		ShutdownCtx:                 shutdownCtx,
 		ShutdownTrigger:             shutdownTrigger,
 		StackDeployer:               stackDeployer,
-		BaseURL:                     *flags.BaseURL,
+		DemoService:                 demoService,
 	}
 }
 
 func main() {
+	configureLogger()
+	setLoggingMode("PRETTY")
+
 	flags := initCLI()
 
-	configureLogger()
+	setLoggingLevel(*flags.LogLevel)
+	setLoggingMode(*flags.LogMode)
 
 	for {
 		server := buildServer(flags)
-		log.Printf("[INFO] [cmd,main] Starting Portainer version %s\n", portainer.APIVersion)
+		log.Info().
+			Str("version", portainer.APIVersion).
+			Str("build_number", build.BuildNumber).
+			Str("image_tag", build.ImageTag).
+			Str("nodejs_version", build.NodejsVersion).
+			Str("yarn_version", build.YarnVersion).
+			Str("webpack_version", build.WebpackVersion).
+			Str("go_version", build.GoVersion).
+			Msg("starting Portainer")
+
 		err := server.Start()
-		log.Printf("[INFO] [cmd,main] Http server exited: %s\n", err)
+		log.Info().Err(err).Msg("HTTP server exited")
 	}
 }

api/cmd/portainer/main_test.go

@@ -21,7 +21,7 @@ func (m mockKingpinSetting) SetValue(value kingpin.Value) {
 func Test_enableFeaturesFromFlags(t *testing.T) {
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	tests := []struct {
@@ -29,11 +29,6 @@ func Test_enableFeaturesFromFlags(t *testing.T) {
 		isSupported bool
 	}{
 		{"test", false},
-		{"openamt", false},
-		{"open-amt", true},
-		{"oPeN-amT", true},
-		{"fdo", true},
-		{"FDO", true},
 	}
 	for _, test := range tests {
 		t.Run(fmt.Sprintf("%s succeeds:%v", test.featureFlag, test.isSupported), func(t *testing.T) {
@@ -81,7 +76,7 @@ func Test_optionalFeature(t *testing.T) {
 
 	is := assert.New(t)
 
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	// Enable the test feature

api/connection.go

@@ -11,14 +11,16 @@ type Connection interface {
 	// write the db contents to filename as json (the schema needs defining)
 	ExportRaw(filename string) error
 
-	//Rollback(force bool) error
-	//MigrateData(migratorParams *database.MigratorParameters, force bool) error
-
 	// TODO: this one is very database specific atm
 	BackupTo(w io.Writer) error
-	GetDatabaseFilename() string
+	GetDatabaseFileName() string
+	GetDatabaseFilePath() string
 	GetStorePath() string
 
+	IsEncryptedStore() bool
+	NeedsEncryptionMigration() (bool, error)
+	SetEncrypted(encrypted bool)
+
 	SetServiceName(bucketName string) error
 	GetObject(bucketName string, key []byte, object interface{}) error
 	UpdateObject(bucketName string, key []byte, object interface{}) error
@@ -27,8 +29,12 @@ type Connection interface {
 	GetNextIdentifier(bucketName string) int
 	CreateObject(bucketName string, fn func(uint64) (int, interface{})) error
 	CreateObjectWithId(bucketName string, id int, obj interface{}) error
+	CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error
 	CreateObjectWithSetSequence(bucketName string, id int, obj interface{}) error
 	GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
 	GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
 	ConvertToKey(v int) []byte
+
+	BackupMetadata() (map[string]interface{}, error)
+	RestoreMetadata(s map[string]interface{}) error
 }

api/crypto/aes_test.go

@@ -7,13 +7,11 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/docker/docker/pkg/ioutils"
 	"github.com/stretchr/testify/assert"
 )
 
 func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "encrypt")
-	defer os.RemoveAll(tmpdir)
+	tmpdir := t.TempDir()
 
 	var (
 		originFilePath    = filepath.Join(tmpdir, "origin")
@@ -52,8 +50,7 @@ func Test_encryptAndDecrypt_withTheSamePassword(t *testing.T) {
 }
 
 func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "encrypt")
-	defer os.RemoveAll(tmpdir)
+	tmpdir := t.TempDir()
 
 	var (
 		originFilePath    = filepath.Join(tmpdir, "origin")
@@ -92,8 +89,7 @@ func Test_encryptAndDecrypt_withEmptyPassword(t *testing.T) {
 }
 
 func Test_decryptWithDifferentPassphrase_shouldProduceWrongResult(t *testing.T) {
-	tmpdir, _ := ioutils.TempDir("", "encrypt")
-	defer os.RemoveAll(tmpdir)
+	tmpdir := t.TempDir()
 
 	var (
 		originFilePath    = filepath.Join(tmpdir, "origin")

api/crypto/hash.go

@@ -9,11 +9,11 @@ type Service struct{}
 
 // Hash hashes a string using the bcrypt algorithm
 func (*Service) Hash(data string) (string, error) {
-	hash, err := bcrypt.GenerateFromPassword([]byte(data), bcrypt.DefaultCost)
+	bytes, err := bcrypt.GenerateFromPassword([]byte(data), bcrypt.DefaultCost)
 	if err != nil {
-		return "", nil
+		return "", err
 	}
-	return string(hash), nil
+	return string(bytes), err
 }
 
 // CompareHashAndData compares a hash to clear data and returns an error if the comparison fails.

api/crypto/hash_test.go

@@ -0,0 +1,53 @@
+package crypto
+
+import (
+	"testing"
+)
+
+func TestService_Hash(t *testing.T) {
+	var s = &Service{}
+
+	type args struct {
+		hash string
+		data string
+	}
+	tests := []struct {
+		name   string
+		args   args
+		expect bool
+	}{
+		{
+			name: "Empty",
+			args: args{
+				hash: "",
+				data: "",
+			},
+			expect: false,
+		},
+		{
+			name: "Matching",
+			args: args{
+				hash: "$2a$10$6BFGd94oYx8k0bFNO6f33uPUpcpAJyg8UVX.akLe9EthF/ZBTXqcy",
+				data: "Passw0rd!",
+			},
+			expect: true,
+		},
+		{
+			name: "Not matching",
+			args: args{
+				hash: "$2a$10$ltKrUZ7492xyutHOb0/XweevU4jyw7QO66rP32jTVOMb3EX3JxA/a",
+				data: "Passw0rd!",
+			},
+			expect: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			err := s.CompareHashAndData(tt.args.hash, tt.args.data)
+			if (err != nil) == tt.expect {
+				t.Errorf("Service.CompareHashAndData() = %v", err)
+			}
+		})
+	}
+}

api/database/boltdb/db.go

@@ -2,6 +2,7 @@ package boltdb
 
 import (
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -9,45 +10,130 @@ import (
 	"path"
 	"time"
 
-	"github.com/boltdb/bolt"
-	"github.com/portainer/portainer/api/dataservices/errors"
+	dserrors "github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
+	bolt "go.etcd.io/bbolt"
 )
 
 const (
-	DatabaseFileName = "portainer.db"
+	DatabaseFileName          = "portainer.db"
+	EncryptedDatabaseFileName = "portainer.edb"
+)
+
+var (
+	ErrHaveEncryptedAndUnencrypted = errors.New("Portainer has detected both an encrypted and un-encrypted database and cannot start.  Only one database should exist")
+	ErrHaveEncryptedWithNoKey      = errors.New("The portainer database is encrypted, but no secret was loaded")
 )
 
 type DbConnection struct {
-	Path string
+	Path            string
+	MaxBatchSize    int
+	MaxBatchDelay   time.Duration
+	InitialMmapSize int
+	EncryptionKey   []byte
+	isEncrypted     bool
 
 	*bolt.DB
 }
 
-func (connection *DbConnection) GetDatabaseFilename() string {
+// GetDatabaseFileName get the database filename
+func (connection *DbConnection) GetDatabaseFileName() string {
+	if connection.IsEncryptedStore() {
+		return EncryptedDatabaseFileName
+	}
+
 	return DatabaseFileName
 }
 
+// GetDataseFilePath get the path + filename for the database file
+func (connection *DbConnection) GetDatabaseFilePath() string {
+	if connection.IsEncryptedStore() {
+		return path.Join(connection.Path, EncryptedDatabaseFileName)
+	}
+
+	return path.Join(connection.Path, DatabaseFileName)
+}
+
+// GetStorePath get the filename and path for the database file
 func (connection *DbConnection) GetStorePath() string {
 	return connection.Path
 }
 
+func (connection *DbConnection) SetEncrypted(flag bool) {
+	connection.isEncrypted = flag
+}
+
+// Return true if the database is encrypted
+func (connection *DbConnection) IsEncryptedStore() bool {
+	return connection.getEncryptionKey() != nil
+}
+
+// NeedsEncryptionMigration returns true if database encryption is enabled and
+// we have an un-encrypted DB that requires migration to an encrypted DB
+func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
+
+	// Cases:  Note, we need to check both portainer.db and portainer.edb
+	// to determine if it's a new store.   We only need to differentiate between cases 2,3 and 5
+
+	// 1) portainer.edb + key     => False
+	// 2) portainer.edb + no key  => ERROR Fatal!
+	// 3) portainer.db  + key     => True  (needs migration)
+	// 4) portainer.db  + no key  => False
+	// 5) NoDB (new)    + key     => False
+	// 6) NoDB (new)    + no key  => False
+	// 7) portainer.db & portainer.edb => ERROR Fatal!
+
+	// If we have a loaded encryption key, always set encrypted
+	if connection.EncryptionKey != nil {
+		connection.SetEncrypted(true)
+	}
+
+	// Check for portainer.db
+	dbFile := path.Join(connection.Path, DatabaseFileName)
+	_, err := os.Stat(dbFile)
+	haveDbFile := err == nil
+
+	// Check for portainer.edb
+	edbFile := path.Join(connection.Path, EncryptedDatabaseFileName)
+	_, err = os.Stat(edbFile)
+	haveEdbFile := err == nil
+
+	if haveDbFile && haveEdbFile {
+		// 7 - encrypted and unencrypted db?
+		return false, ErrHaveEncryptedAndUnencrypted
+	}
+
+	if haveDbFile && connection.EncryptionKey != nil {
+		// 3 - needs migration
+		return true, nil
+	}
+
+	if haveEdbFile && connection.EncryptionKey == nil {
+		// 2 - encrypted db, but no key?
+		return false, ErrHaveEncryptedWithNoKey
+	}
+
+	// 1, 4, 5, 6
+	return false, nil
+}
+
 // Open opens and initializes the BoltDB database.
 func (connection *DbConnection) Open() error {
 
-	// Disabled for now.  Can't use feature flags due to the way that works
-	// databaseExportPath := path.Join(connection.Path, fmt.Sprintf("raw-%s-%d.json", DatabaseFileName, time.Now().Unix()))
-	// if err := connection.ExportRaw(databaseExportPath); err != nil {
-	// 	log.Printf("raw export to %s error: %s", databaseExportPath, err)
-	// } else {
-	// 	log.Printf("raw export to %s success", databaseExportPath)
-	// }
+	log.Info().Str("filename", connection.GetDatabaseFileName()).Msg("loading PortainerDB")
 
-	databasePath := path.Join(connection.Path, DatabaseFileName)
-
-	db, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	// Now we open the db
+	databasePath := connection.GetDatabaseFilePath()
+	db, err := bolt.Open(databasePath, 0600, &bolt.Options{
+		Timeout:         1 * time.Second,
+		InitialMmapSize: connection.InitialMmapSize,
+	})
 	if err != nil {
 		return err
 	}
+	db.MaxBatchSize = connection.MaxBatchSize
+	db.MaxBatchDelay = connection.MaxBatchDelay
 	connection.DB = db
 	return nil
 }
@@ -71,12 +157,12 @@ func (connection *DbConnection) BackupTo(w io.Writer) error {
 }
 
 func (connection *DbConnection) ExportRaw(filename string) error {
-	databasePath := path.Join(connection.Path, DatabaseFileName)
+	databasePath := connection.GetDatabaseFilePath()
 	if _, err := os.Stat(databasePath); err != nil {
 		return fmt.Errorf("stat on %s failed: %s", databasePath, err)
 	}
 
-	b, err := exportJson(databasePath)
+	b, err := connection.ExportJson(databasePath, true)
 	if err != nil {
 		return err
 	}
@@ -94,12 +180,9 @@ func (connection *DbConnection) ConvertToKey(v int) []byte {
 
 // CreateBucket is a generic function used to create a bucket inside a database database.
 func (connection *DbConnection) SetServiceName(bucketName string) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
-		if err != nil {
-			return err
-		}
-		return nil
+		return err
 	})
 }
 
@@ -112,7 +195,7 @@ func (connection *DbConnection) GetObject(bucketName string, key []byte, object
 
 		value := bucket.Get(key)
 		if value == nil {
-			return errors.ErrObjectNotFound
+			return dserrors.ErrObjectNotFound
 		}
 
 		data = make([]byte, len(value))
@@ -124,31 +207,33 @@ func (connection *DbConnection) GetObject(bucketName string, key []byte, object
 		return err
 	}
 
-	return UnmarshalObject(data, object)
+	return connection.UnmarshalObjectWithJsoniter(data, object)
+}
+
+func (connection *DbConnection) getEncryptionKey() []byte {
+	if !connection.isEncrypted {
+		return nil
+	}
+
+	return connection.EncryptionKey
 }
 
 // UpdateObject is a generic function used to update an object inside a database database.
 func (connection *DbConnection) UpdateObject(bucketName string, key []byte, object interface{}) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	data, err := connection.MarshalObject(object)
+	if err != nil {
+		return err
+	}
+
+	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
-
-		data, err := MarshalObject(object)
-		if err != nil {
-			return err
-		}
-
-		err = bucket.Put(key, data)
-		if err != nil {
-			return err
-		}
-
-		return nil
+		return bucket.Put(key, data)
 	})
 }
 
 // DeleteObject is a generic function used to delete an object inside a database database.
 func (connection *DbConnection) DeleteObject(bucketName string, key []byte) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		return bucket.Delete(key)
 	})
@@ -157,13 +242,13 @@ func (connection *DbConnection) DeleteObject(bucketName string, key []byte) erro
 // DeleteAllObjects delete all objects where matching() returns (id, ok).
 // TODO: think about how to return the error inside (maybe change ok to type err, and use "notfound"?
 func (connection *DbConnection) DeleteAllObjects(bucketName string, matching func(o interface{}) (id int, ok bool)) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		cursor := bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
 			var obj interface{}
-			err := UnmarshalObject(v, &obj)
+			err := connection.UnmarshalObject(v, &obj)
 			if err != nil {
 				return err
 			}
@@ -184,7 +269,7 @@ func (connection *DbConnection) DeleteAllObjects(bucketName string, matching fun
 func (connection *DbConnection) GetNextIdentifier(bucketName string) int {
 	var identifier int
 
-	connection.Update(func(tx *bolt.Tx) error {
+	connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		id, err := bucket.NextSequence()
 		if err != nil {
@@ -199,13 +284,13 @@ func (connection *DbConnection) GetNextIdentifier(bucketName string) int {
 
 // CreateObject creates a new object in the bucket, using the next bucket sequence id
 func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64) (int, interface{})) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		seqId, _ := bucket.NextSequence()
 		id, obj := fn(seqId)
 
-		data, err := MarshalObject(obj)
+		data, err := connection.MarshalObject(obj)
 		if err != nil {
 			return err
 		}
@@ -216,10 +301,9 @@ func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64)
 
 // CreateObjectWithId creates a new object in the bucket, using the specified id
 func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, obj interface{}) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
-
-		data, err := MarshalObject(obj)
+		data, err := connection.MarshalObject(obj)
 		if err != nil {
 			return err
 		}
@@ -228,10 +312,23 @@ func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, ob
 	})
 }
 
+// CreateObjectWithStringId creates a new object in the bucket, using the specified id
+func (connection *DbConnection) CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket([]byte(bucketName))
+		data, err := connection.MarshalObject(obj)
+		if err != nil {
+			return err
+		}
+
+		return bucket.Put(id, data)
+	})
+}
+
 // CreateObjectWithSetSequence creates a new object in the bucket, using the specified id, and sets the bucket sequence
 // avoid this :)
 func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, id int, obj interface{}) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		// We manually manage sequences for schedules
@@ -240,7 +337,7 @@ func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, i
 			return err
 		}
 
-		data, err := MarshalObject(obj)
+		data, err := connection.MarshalObject(obj)
 		if err != nil {
 			return err
 		}
@@ -255,7 +352,7 @@ func (connection *DbConnection) GetAll(bucketName string, obj interface{}, appen
 
 		cursor := bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			err := UnmarshalObject(v, obj)
+			err := connection.UnmarshalObject(v, obj)
 			if err != nil {
 				return err
 			}
@@ -267,6 +364,7 @@ func (connection *DbConnection) GetAll(bucketName string, obj interface{}, appen
 
 		return nil
 	})
+
 	return err
 }
 
@@ -277,7 +375,7 @@ func (connection *DbConnection) GetAllWithJsoniter(bucketName string, obj interf
 
 		cursor := bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			err := UnmarshalObjectWithJsoniter(v, obj)
+			err := connection.UnmarshalObjectWithJsoniter(v, obj)
 			if err != nil {
 				return err
 			}
@@ -291,3 +389,44 @@ func (connection *DbConnection) GetAllWithJsoniter(bucketName string, obj interf
 	})
 	return err
 }
+
+func (connection *DbConnection) BackupMetadata() (map[string]interface{}, error) {
+	buckets := map[string]interface{}{}
+
+	err := connection.View(func(tx *bolt.Tx) error {
+		err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
+			bucketName := string(name)
+			bucket = tx.Bucket([]byte(bucketName))
+			seqId := bucket.Sequence()
+			buckets[bucketName] = int(seqId)
+			return nil
+		})
+
+		return err
+	})
+
+	return buckets, err
+}
+
+func (connection *DbConnection) RestoreMetadata(s map[string]interface{}) error {
+	var err error
+
+	for bucketName, v := range s {
+		id, ok := v.(float64) // JSON ints are unmarshalled to interface as float64. See: https://pkg.go.dev/encoding/json#Decoder.Decode
+		if !ok {
+			log.Error().Str("bucket", bucketName).Msg("failed to restore metadata to bucket, skipped")
+			continue
+		}
+
+		err = connection.Batch(func(tx *bolt.Tx) error {
+			bucket, err := tx.CreateBucketIfNotExists([]byte(bucketName))
+			if err != nil {
+				return err
+			}
+
+			return bucket.SetSequence(uint64(id))
+		})
+	}
+
+	return err
+}

api/database/boltdb/db_test.go

@@ -0,0 +1,124 @@
+package boltdb
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_NeedsEncryptionMigration(t *testing.T) {
+	// Test the specific scenarios mentioned in NeedsEncryptionMigration
+
+	// i.e.
+	// Cases:  Note, we need to check both portainer.db and portainer.edb
+	// to determine if it's a new store.   We only need to differentiate between cases 2,3 and 5
+
+	// 1) portainer.edb + key     => False
+	// 2) portainer.edb + no key  => ERROR Fatal!
+	// 3) portainer.db  + key     => True  (needs migration)
+	// 4) portainer.db  + no key  => False
+	// 5) NoDB (new)    + key     => False
+	// 6) NoDB (new)    + no key  => False
+	// 7) portainer.db & portainer.edb (key not important) => ERROR Fatal!
+
+	is := assert.New(t)
+	dir := t.TempDir()
+
+	cases := []struct {
+		name         string
+		dbname       string
+		key          bool
+		expectError  error
+		expectResult bool
+	}{
+		{
+			name:         "portainer.edb + key",
+			dbname:       EncryptedDatabaseFileName,
+			key:          true,
+			expectError:  nil,
+			expectResult: false,
+		},
+		{
+			name:         "portainer.db + key (migration needed)",
+			dbname:       DatabaseFileName,
+			key:          true,
+			expectError:  nil,
+			expectResult: true,
+		},
+		{
+			name:         "portainer.db + no key",
+			dbname:       DatabaseFileName,
+			key:          false,
+			expectError:  nil,
+			expectResult: false,
+		},
+		{
+			name:         "NoDB (new) + key",
+			dbname:       "",
+			key:          false,
+			expectError:  nil,
+			expectResult: false,
+		},
+		{
+			name:         "NoDB (new) + no key",
+			dbname:       "",
+			key:          false,
+			expectError:  nil,
+			expectResult: false,
+		},
+
+		// error tests
+		{
+			name:         "portainer.edb + no key",
+			dbname:       EncryptedDatabaseFileName,
+			key:          false,
+			expectError:  ErrHaveEncryptedWithNoKey,
+			expectResult: false,
+		},
+		{
+			name:         "portainer.db & portainer.edb",
+			dbname:       "both",
+			key:          true,
+			expectError:  ErrHaveEncryptedAndUnencrypted,
+			expectResult: false,
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+
+			connection := DbConnection{Path: dir}
+
+			if tc.dbname == "both" {
+				// Special case.  If portainer.db and portainer.edb exist.
+				dbFile1 := path.Join(connection.Path, DatabaseFileName)
+				f, _ := os.Create(dbFile1)
+				f.Close()
+				defer os.Remove(dbFile1)
+
+				dbFile2 := path.Join(connection.Path, EncryptedDatabaseFileName)
+				f, _ = os.Create(dbFile2)
+				f.Close()
+				defer os.Remove(dbFile2)
+			} else if tc.dbname != "" {
+				dbFile := path.Join(connection.Path, tc.dbname)
+				f, _ := os.Create(dbFile)
+				f.Close()
+				defer os.Remove(dbFile)
+			}
+
+			if tc.key {
+				connection.EncryptionKey = []byte("secret")
+			}
+
+			result, err := connection.NeedsEncryptionMigration()
+
+			is.Equal(tc.expectError, err, "Fatal Error failure. Test: %s", tc.name)
+			is.Equal(result, tc.expectResult, "Failed test: %s", tc.name)
+		})
+	}
+}

api/database/boltdb/export.go

@@ -4,14 +4,36 @@ import (
 	"encoding/json"
 	"time"
 
-	"github.com/boltdb/bolt"
-	"github.com/sirupsen/logrus"
+	"github.com/rs/zerolog/log"
+	bolt "go.etcd.io/bbolt"
 )
 
+func backupMetadata(connection *bolt.DB) (map[string]interface{}, error) {
+	buckets := map[string]interface{}{}
+
+	err := connection.View(func(tx *bolt.Tx) error {
+		err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
+			bucketName := string(name)
+			bucket = tx.Bucket([]byte(bucketName))
+			seqId := bucket.Sequence()
+			buckets[bucketName] = int(seqId)
+			return nil
+		})
+
+		return err
+	})
+
+	return buckets, err
+}
+
+// ExportJSON creates a JSON representation from a DbConnection. You can include
+// the database's metadata or ignore it. Ensure the database is closed before
+// using this function.
 // inspired by github.com/konoui/boltdb-exporter (which has no license)
 // but very much simplified, based on how we use boltdb
+func (c *DbConnection) ExportJson(databasePath string, metadata bool) ([]byte, error) {
+	log.Debug().Str("databasePath", databasePath).Msg("exportJson")
 
-func exportJson(databasePath string) ([]byte, error) {
 	connection, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second, ReadOnly: true})
 	if err != nil {
 		return []byte("{}"), err
@@ -19,6 +41,14 @@ func exportJson(databasePath string) ([]byte, error) {
 	defer connection.Close()
 
 	backup := make(map[string]interface{})
+	if metadata {
+		meta, err := backupMetadata(connection)
+		if err != nil {
+			log.Error().Err(err).Msg("failed exporting metadata")
+		}
+
+		backup["__metadata"] = meta
+	}
 
 	err = connection.View(func(tx *bolt.Tx) error {
 		err = tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
@@ -30,35 +60,51 @@ func exportJson(databasePath string) ([]byte, error) {
 				if v == nil {
 					continue
 				}
+
 				var obj interface{}
-				err := UnmarshalObject(v, &obj)
+				err := c.UnmarshalObject(v, &obj)
 				if err != nil {
-					logrus.WithError(err).Errorf("Failed to unmarshal (bucket %s): %v", bucketName, string(v))
+					log.Error().
+						Str("bucket", bucketName).
+						Str("object", string(v)).
+						Err(err).
+						Msg("failed to unmarshal")
+
 					obj = v
 				}
+
 				if bucketName == "version" {
 					version[string(k)] = string(v)
 				} else {
 					list = append(list, obj)
 				}
 			}
+
 			if bucketName == "version" {
 				backup[bucketName] = version
+				return nil
 			}
+
 			if len(list) > 0 {
 				if bucketName == "ssl" ||
 					bucketName == "settings" ||
 					bucketName == "tunnel_server" {
-					backup[bucketName] = list[0]
+					backup[bucketName] = nil
+					if len(list) > 0 {
+						backup[bucketName] = list[0]
+					}
 					return nil
 				}
 				backup[bucketName] = list
+				return nil
 			}
 
 			return nil
 		})
+
 		return err
 	})
+
 	if err != nil {
 		return []byte("{}"), err
 	}

api/database/boltdb/json.go

@@ -1,26 +1,72 @@
 package boltdb
 
 import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
 	"encoding/json"
+	"fmt"
+	"io"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/pkg/errors"
 )
 
+var errEncryptedStringTooShort = fmt.Errorf("encrypted string too short")
+
 // MarshalObject encodes an object to binary format
-func MarshalObject(object interface{}) ([]byte, error) {
+func (connection *DbConnection) MarshalObject(object interface{}) (data []byte, err error) {
 	// Special case for the VERSION bucket. Here we're not using json
 	if v, ok := object.(string); ok {
-		return []byte(v), nil
+		data = []byte(v)
+	} else {
+		data, err = json.Marshal(object)
+		if err != nil {
+			return data, err
+		}
 	}
-
-	return json.Marshal(object)
+	if connection.getEncryptionKey() == nil {
+		return data, nil
+	}
+	return encrypt(data, connection.getEncryptionKey())
 }
 
 // UnmarshalObject decodes an object from binary data
-func UnmarshalObject(data []byte, object interface{}) error {
-	// Special case for the VERSION bucket. Here we're not using json
-	// So we need to return it as a string
-	err := json.Unmarshal(data, object)
+func (connection *DbConnection) UnmarshalObject(data []byte, object interface{}) error {
+	var err error
+	if connection.getEncryptionKey() != nil {
+		data, err = decrypt(data, connection.getEncryptionKey())
+		if err != nil {
+			return errors.Wrap(err, "Failed decrypting object")
+		}
+	}
+	e := json.Unmarshal(data, object)
+	if e != nil {
+		// Special case for the VERSION bucket. Here we're not using json
+		// So we need to return it as a string
+		s, ok := object.(*string)
+		if !ok {
+			return errors.Wrap(err, e.Error())
+		}
+
+		*s = string(data)
+	}
+	return err
+}
+
+// UnmarshalObjectWithJsoniter decodes an object from binary data
+// using the jsoniter library. It is mainly used to accelerate environment(endpoint)
+// decoding at the moment.
+func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
+	if connection.getEncryptionKey() != nil {
+		var err error
+		data, err = decrypt(data, connection.getEncryptionKey())
+		if err != nil {
+			return err
+		}
+	}
+	var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
+	err := jsoni.Unmarshal(data, &object)
 	if err != nil {
 		if s, ok := object.(*string); ok {
 			*s = string(data)
@@ -33,10 +79,55 @@ func UnmarshalObject(data []byte, object interface{}) error {
 	return nil
 }
 
-// UnmarshalObjectWithJsoniter decodes an object from binary data
-// using the jsoniter library. It is mainly used to accelerate environment(endpoint)
-// decoding at the moment.
-func UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
-	var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
-	return jsoni.Unmarshal(data, &object)
+// mmm, don't have a KMS .... aes GCM seems the most likely from
+// https://gist.github.com/atoponce/07d8d4c833873be2f68c34f9afc5a78a#symmetric-encryption
+
+func encrypt(plaintext []byte, passphrase []byte) (encrypted []byte, err error) {
+	block, _ := aes.NewCipher(passphrase)
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return encrypted, err
+	}
+	nonce := make([]byte, gcm.NonceSize())
+	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+		return encrypted, err
+	}
+	ciphertextByte := gcm.Seal(
+		nonce,
+		nonce,
+		plaintext,
+		nil)
+	return ciphertextByte, nil
+}
+
+func decrypt(encrypted []byte, passphrase []byte) (plaintextByte []byte, err error) {
+	if string(encrypted) == "false" {
+		return []byte("false"), nil
+	}
+	block, err := aes.NewCipher(passphrase)
+	if err != nil {
+		return encrypted, errors.Wrap(err, "Error creating cypher block")
+	}
+
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return encrypted, errors.Wrap(err, "Error creating GCM")
+	}
+
+	nonceSize := gcm.NonceSize()
+	if len(encrypted) < nonceSize {
+		return encrypted, errEncryptedStringTooShort
+	}
+
+	nonce, ciphertextByteClean := encrypted[:nonceSize], encrypted[nonceSize:]
+	plaintextByte, err = gcm.Open(
+		nil,
+		nonce,
+		ciphertextByteClean,
+		nil)
+	if err != nil {
+		return encrypted, errors.Wrap(err, "Error decrypting text")
+	}
+
+	return plaintextByte, err
 }

api/database/boltdb/json_test.go

@@ -1,6 +1,7 @@
 package boltdb
 
 import (
+	"crypto/sha256"
 	"fmt"
 	"testing"
 
@@ -8,9 +9,17 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-const jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","Credentials":{"MPSUser":"","MPSPassword":"","MPSToken":""},"DomainConfiguration":{"CertFileText":"","CertPassword":"","DomainName":""},"WirelessConfiguration":null},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","EnableTelemetry":true,"HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
+const (
+	jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"InternalAuthSettings": {"RequiredPasswordLength": 12}"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","MPSUser":"","MPSPassword":"","MPSToken":"","CertFileContent":"","CertFileName":"","CertFilePassword":"","DomainName":""},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","EnableTelemetry":true,"HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
+	passphrase = "my secret key"
+)
 
-func Test_MarshalObject(t *testing.T) {
+func secretToEncryptionKey(passphrase string) []byte {
+	hash := sha256.Sum256([]byte(passphrase))
+	return hash[:]
+}
+
+func Test_MarshalObjectUnencrypted(t *testing.T) {
 	is := assert.New(t)
 
 	uuid := uuid.Must(uuid.NewV4())
@@ -73,16 +82,18 @@ func Test_MarshalObject(t *testing.T) {
 		},
 	}
 
+	conn := DbConnection{}
+
 	for _, test := range tests {
 		t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
-			data, err := MarshalObject(test.object)
+			data, err := conn.MarshalObject(test.object)
 			is.NoError(err)
 			is.Equal(test.expected, string(data))
 		})
 	}
 }
 
-func Test_UnMarshalObject(t *testing.T) {
+func Test_UnMarshalObjectUnencrypted(t *testing.T) {
 	is := assert.New(t)
 
 	// Based on actual data entering and what we expect out of the function
@@ -105,18 +116,62 @@ func Test_UnMarshalObject(t *testing.T) {
 			expected: "9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6",
 		},
 		{
-			// An unmarshalled json object string should return the same as a string without error also
+			// An un-marshalled json object string should return the same as a string without error also
 			object:   []byte(jsonobject),
 			expected: jsonobject,
 		},
 	}
 
+	conn := DbConnection{}
+
 	for _, test := range tests {
 		t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
 			var object string
-			err := UnmarshalObject(test.object, &object)
+			err := conn.UnmarshalObject(test.object, &object)
 			is.NoError(err)
 			is.Equal(test.expected, string(object))
 		})
 	}
 }
+
+func Test_ObjectMarshallingEncrypted(t *testing.T) {
+	is := assert.New(t)
+
+	// Based on actual data entering and what we expect out of the function
+
+	tests := []struct {
+		object   []byte
+		expected string
+	}{
+		{
+			object: []byte(""),
+		},
+		{
+			object: []byte("35"),
+		},
+		{
+			// An unmarshalled byte string should return the same without error
+			object: []byte("9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6"),
+		},
+		{
+			// An un-marshalled json object string should return the same as a string without error also
+			object: []byte(jsonobject),
+		},
+	}
+
+	key := secretToEncryptionKey(passphrase)
+	conn := DbConnection{EncryptionKey: key}
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
+
+			data, err := conn.MarshalObject(test.object)
+			is.NoError(err)
+
+			var object []byte
+			err = conn.UnmarshalObject(data, &object)
+
+			is.NoError(err)
+			is.Equal(test.object, object)
+		})
+	}
+}

api/database/database.go

@@ -2,15 +2,19 @@ package database
 
 import (
 	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/boltdb"
 )
 
 // NewDatabase should use config options to return a connection to the requested database
-func NewDatabase(storeType, storePath string) (connection portainer.Connection, err error) {
+func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection portainer.Connection, err error) {
 	switch storeType {
 	case "boltdb":
-		return &boltdb.DbConnection{Path: storePath}, nil
+		return &boltdb.DbConnection{
+			Path:          storePath,
+			EncryptionKey: encryptionKey,
+		}, nil
 	}
-	return nil, fmt.Errorf("Unknown storage database: %s", storeType)
+	return nil, fmt.Errorf("unknown storage database: %s", storeType)
 }

api/database/models/configmaps_and_secrets.go

@@ -0,0 +1,17 @@
+package models
+
+type (
+	K8sConfigMapOrSecret struct {
+		UID          string            `json:"UID"`
+		Name         string            `json:"Name"`
+		Namespace    string            `json:"Namespace"`
+		CreationDate string            `json:"CreationDate"`
+		Annotations  map[string]string `json:"Annotations"`
+		Data         map[string]string `json:"Data"`
+		Applications []string          `json:"Applications"`
+		IsSecret     bool              `json:"IsSecret"`
+
+		// SecretType will be an empty string for config maps.
+		SecretType string `json:"SecretType"`
+	}
+)

api/database/models/ingress.go

@@ -0,0 +1,75 @@
+package models
+
+import (
+	"errors"
+	"net/http"
+)
+
+type (
+	K8sIngressController struct {
+		Name         string `json:"Name"`
+		ClassName    string `json:"ClassName"`
+		Type         string `json:"Type"`
+		Availability bool   `json:"Availability"`
+		New          bool   `json:"New"`
+		Used         bool   `json:"Used"`
+	}
+
+	K8sIngressControllers []K8sIngressController
+
+	K8sIngressInfo struct {
+		Name        string            `json:"Name"`
+		UID         string            `json:"UID"`
+		Type        string            `json:"Type"`
+		Namespace   string            `json:"Namespace"`
+		ClassName   string            `json:"ClassName"`
+		Annotations map[string]string `json:"Annotations"`
+		Hosts       []string          `json:"Hosts"`
+		Paths       []K8sIngressPath  `json:"Paths"`
+		TLS         []K8sIngressTLS   `json:"TLS"`
+	}
+
+	K8sIngressTLS struct {
+		Hosts      []string `json:"Hosts"`
+		SecretName string   `json:"SecretName"`
+	}
+
+	K8sIngressPath struct {
+		IngressName string `json:"IngressName"`
+		Host        string `json:"Host"`
+		ServiceName string `json:"ServiceName"`
+		Port        int    `json:"Port"`
+		Path        string `json:"Path"`
+		PathType    string `json:"PathType"`
+	}
+
+	// K8sIngressDeleteRequests is a mapping of namespace names to a slice of
+	// ingress names.
+	K8sIngressDeleteRequests map[string][]string
+)
+
+func (r K8sIngressControllers) Validate(request *http.Request) error {
+	return nil
+}
+
+func (r K8sIngressInfo) Validate(request *http.Request) error {
+	if r.Name == "" {
+		return errors.New("missing ingress name from the request payload")
+	}
+	if r.Namespace == "" {
+		return errors.New("missing ingress Namespace from the request payload")
+	}
+	return nil
+}
+
+func (r K8sIngressDeleteRequests) Validate(request *http.Request) error {
+	if len(r) == 0 {
+		return errors.New("missing deletion request list in payload")
+	}
+	for ns := range r {
+		if len(ns) == 0 {
+			return errors.New("deletion given with empty namespace")
+		}
+	}
+	return nil
+}

api/database/models/namespaces.go

@@ -0,0 +1,12 @@
+package models
+
+import "net/http"
+
+type K8sNamespaceDetails struct {
+	Name        string            `json:"Name"`
+	Annotations map[string]string `json:"Annotations"`
+}
+
+func (r *K8sNamespaceDetails) Validate(request *http.Request) error {
+	return nil
+}

api/database/models/services.go

@@ -0,0 +1,64 @@
+package models
+
+import (
+	"errors"
+	"net/http"
+)
+
+type (
+	K8sServiceInfo struct {
+		Name                          string              `json:"Name"`
+		UID                           string              `json:"UID"`
+		Type                          string              `json:"Type"`
+		Namespace                     string              `json:"Namespace"`
+		Annotations                   map[string]string   `json:"Annotations"`
+		CreationTimestamp             string              `json:"CreationTimestamp"`
+		Labels                        map[string]string   `json:"Labels"`
+		AllocateLoadBalancerNodePorts *bool               `json:"AllocateLoadBalancerNodePorts,omitempty"`
+		Ports                         []K8sServicePort    `json:"Ports"`
+		Selector                      map[string]string   `json:"Selector"`
+		IngressStatus                 []K8sServiceIngress `json:"IngressStatus"`
+	}
+
+	K8sServicePort struct {
+		Name       string `json:"Name"`
+		NodePort   int    `json:"NodePort"`
+		Port       int    `json:"Port"`
+		Protocol   string `json:"Protocol"`
+		TargetPort int    `json:"TargetPort"`
+	}
+
+	K8sServiceIngress struct {
+		IP   string `json:"IP"`
+		Host string `json:"Host"`
+	}
+
+	// K8sServiceDeleteRequests is a mapping of namespace names to a slice of
+	// service names.
+	K8sServiceDeleteRequests map[string][]string
+)
+
+func (s *K8sServiceInfo) Validate(request *http.Request) error {
+	if s.Name == "" {
+		return errors.New("missing service name from the request payload")
+	}
+	if s.Namespace == "" {
+		return errors.New("missing service namespace from the request payload")
+	}
+	if s.Ports == nil {
+		return errors.New("missing service ports from the request payload")
+	}
+	return nil
+}
+
+func (r K8sServiceDeleteRequests) Validate(request *http.Request) error {
+	if len(r) == 0 {
+		return errors.New("missing deletion request list in payload")
+	}
+	for ns := range r {
+		if len(ns) == 0 {
+			return errors.New("deletion given with empty namespace")
+		}
+	}
+	return nil
+}

api/dataservices/apikeyrepository/apikeyrepository.go

@@ -6,7 +6,8 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -41,12 +42,14 @@ func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer
 		func(obj interface{}) (interface{}, error) {
 			record, ok := obj.(*portainer.APIKey)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to APIKey object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
 				return nil, fmt.Errorf("Failed to convert to APIKey object: %s", obj)
 			}
+
 			if record.UserID == userID {
 				result = append(result, *record)
 			}
+
 			return &portainer.APIKey{}, nil
 		})
 
@@ -64,18 +67,21 @@ func (service *Service) GetAPIKeyByDigest(digest []byte) (*portainer.APIKey, err
 		func(obj interface{}) (interface{}, error) {
 			key, ok := obj.(*portainer.APIKey)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to APIKey object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
 				return nil, fmt.Errorf("Failed to convert to APIKey object: %s", obj)
 			}
 			if bytes.Equal(key.Digest, digest) {
 				k = key
 				return nil, stop
 			}
+
 			return &portainer.APIKey{}, nil
 		})
+
 	if err == stop {
 		return k, nil
 	}
+
 	if err == nil {
 		return nil, errors.ErrObjectNotFound
 	}

api/dataservices/customtemplate/customtemplate.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -44,10 +45,11 @@ func (service *Service) CustomTemplates() ([]portainer.CustomTemplate, error) {
 			//var tag portainer.Tag
 			customTemplate, ok := obj.(*portainer.CustomTemplate)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to CustomTemplate object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to CustomTemplate object")
 				return nil, fmt.Errorf("Failed to convert to CustomTemplate object: %s", obj)
 			}
 			customTemplates = append(customTemplates, *customTemplate)
+
 			return &portainer.CustomTemplate{}, nil
 		})
 

api/dataservices/edgegroup/edgegroup.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -43,10 +44,11 @@ func (service *Service) EdgeGroups() ([]portainer.EdgeGroup, error) {
 		func(obj interface{}) (interface{}, error) {
 			group, ok := obj.(*portainer.EdgeGroup)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to EdgeGroup object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeGroup object")
 				return nil, fmt.Errorf("Failed to convert to EdgeGroup object: %s", obj)
 			}
 			groups = append(groups, *group)
+
 			return &portainer.EdgeGroup{}, nil
 		})
 

api/dataservices/edgejob/edgejob.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -41,13 +42,14 @@ func (service *Service) EdgeJobs() ([]portainer.EdgeJob, error) {
 		BucketName,
 		&portainer.EdgeJob{},
 		func(obj interface{}) (interface{}, error) {
-			//var tag portainer.Tag
 			job, ok := obj.(*portainer.EdgeJob)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to EdgeJob object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeJob object")
 				return nil, fmt.Errorf("Failed to convert to EdgeJob object: %s", obj)
 			}
+
 			edgeJobs = append(edgeJobs, *job)
+
 			return &portainer.EdgeJob{}, nil
 		})
 
@@ -67,14 +69,14 @@ func (service *Service) EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, err
 	return &edgeJob, nil
 }
 
-// CreateEdgeJob creates a new Edge job
-func (service *Service) Create(edgeJob *portainer.EdgeJob) error {
-	return service.connection.CreateObject(
+// Create creates a new EdgeJob
+func (service *Service) Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error {
+	edgeJob.ID = ID
+
+	return service.connection.CreateObjectWithId(
 		BucketName,
-		func(id uint64) (int, interface{}) {
-			edgeJob.ID = portainer.EdgeJobID(id)
-			return int(edgeJob.ID), edgeJob
-		},
+		int(edgeJob.ID),
+		edgeJob,
 	)
 }
 

api/dataservices/edgestack/edgestack.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -44,10 +45,12 @@ func (service *Service) EdgeStacks() ([]portainer.EdgeStack, error) {
 			//var tag portainer.Tag
 			stack, ok := obj.(*portainer.EdgeStack)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to EdgeStack object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeStack object")
 				return nil, fmt.Errorf("Failed to convert to EdgeStack object: %s", obj)
 			}
+
 			stacks = append(stacks, *stack)
+
 			return &portainer.EdgeStack{}, nil
 		})
 
@@ -67,14 +70,15 @@ func (service *Service) EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStac
 	return &stack, nil
 }
 
-// CreateEdgeStack assign an ID to a new Edge stack and saves it.
-func (service *Service) Create(edgeStack *portainer.EdgeStack) error {
-	return service.connection.CreateObject(
+// CreateEdgeStack saves an Edge stack object to db.
+func (service *Service) Create(id portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error {
+
+	edgeStack.ID = id
+
+	return service.connection.CreateObjectWithId(
 		BucketName,
-		func(id uint64) (int, interface{}) {
-			edgeStack.ID = portainer.EdgeStackID(id)
-			return int(edgeStack.ID), edgeStack
-		},
+		int(edgeStack.ID),
+		edgeStack,
 	)
 }
 

api/dataservices/edgeupdateschedule/edgeupdateschedule.go

@@ -0,0 +1,185 @@
+package edgeupdateschedule
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/pkg/errors"
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/edgetypes"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "edge_update_schedule"
+)
+
+// Service represents a service for managing Edge Update Schedule data.
+type Service struct {
+	connection portainer.Connection
+
+	mu                 sync.Mutex
+	idxActiveSchedules map[portainer.EndpointID]*edgetypes.EndpointUpdateScheduleRelation
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	service := &Service{
+		connection: connection,
+	}
+
+	service.idxActiveSchedules = map[portainer.EndpointID]*edgetypes.EndpointUpdateScheduleRelation{}
+
+	schedules, err := service.List()
+	if err != nil {
+		return nil, errors.WithMessage(err, "Unable to list schedules")
+	}
+
+	for _, schedule := range schedules {
+		service.setRelation(&schedule)
+	}
+
+	return service, nil
+}
+
+func (service *Service) ActiveSchedule(environmentID portainer.EndpointID) *edgetypes.EndpointUpdateScheduleRelation {
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
+	return service.idxActiveSchedules[environmentID]
+}
+
+func (service *Service) ActiveSchedules(environmentsIDs []portainer.EndpointID) []edgetypes.EndpointUpdateScheduleRelation {
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
+	schedules := []edgetypes.EndpointUpdateScheduleRelation{}
+
+	for _, environmentID := range environmentsIDs {
+		if s, ok := service.idxActiveSchedules[environmentID]; ok {
+			schedules = append(schedules, *s)
+		}
+	}
+
+	return schedules
+}
+
+// List return an array containing all the items in the bucket.
+func (service *Service) List() ([]edgetypes.UpdateSchedule, error) {
+	var list = make([]edgetypes.UpdateSchedule, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&edgetypes.UpdateSchedule{},
+		func(obj interface{}) (interface{}, error) {
+			item, ok := obj.(*edgetypes.UpdateSchedule)
+			if !ok {
+				logrus.WithField("obj", obj).Errorf("Failed to convert to EdgeUpdateSchedule object")
+				return nil, fmt.Errorf("failed to convert to EdgeUpdateSchedule object: %s", obj)
+			}
+			list = append(list, *item)
+			return &edgetypes.UpdateSchedule{}, nil
+		})
+
+	return list, err
+}
+
+// Item returns a item by ID.
+func (service *Service) Item(ID edgetypes.UpdateScheduleID) (*edgetypes.UpdateSchedule, error) {
+	var item edgetypes.UpdateSchedule
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.GetObject(BucketName, identifier, &item)
+	if err != nil {
+		return nil, err
+	}
+
+	return &item, nil
+}
+
+// Create assign an ID to a new object and saves it.
+func (service *Service) Create(item *edgetypes.UpdateSchedule) error {
+	err := service.connection.CreateObject(
+		BucketName,
+		func(id uint64) (int, interface{}) {
+			item.ID = edgetypes.UpdateScheduleID(id)
+			return int(item.ID), item
+		},
+	)
+
+	if err != nil {
+		return err
+	}
+
+	return service.setRelation(item)
+}
+
+// Update updates an item.
+func (service *Service) Update(id edgetypes.UpdateScheduleID, item *edgetypes.UpdateSchedule) error {
+	identifier := service.connection.ConvertToKey(int(id))
+	err := service.connection.UpdateObject(BucketName, identifier, item)
+	if err != nil {
+		return err
+	}
+
+	service.cleanRelation(id)
+
+	return service.setRelation(item)
+}
+
+// Delete deletes an item.
+func (service *Service) Delete(id edgetypes.UpdateScheduleID) error {
+
+	service.cleanRelation(id)
+
+	identifier := service.connection.ConvertToKey(int(id))
+	return service.connection.DeleteObject(BucketName, identifier)
+}
+
+func (service *Service) cleanRelation(id edgetypes.UpdateScheduleID) {
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
+	for _, schedule := range service.idxActiveSchedules {
+		if schedule != nil && schedule.ScheduleID == id {
+			delete(service.idxActiveSchedules, schedule.EnvironmentID)
+		}
+	}
+}
+
+func (service *Service) setRelation(schedule *edgetypes.UpdateSchedule) error {
+	service.mu.Lock()
+	defer service.mu.Unlock()
+
+	for environmentID, environmentStatus := range schedule.Status {
+		if environmentStatus.Status != edgetypes.UpdateScheduleStatusPending {
+			continue
+		}
+
+		// this should never happen
+		if service.idxActiveSchedules[environmentID] != nil && service.idxActiveSchedules[environmentID].ScheduleID != schedule.ID {
+			return errors.New("Multiple schedules are pending for the same environment")
+		}
+
+		service.idxActiveSchedules[environmentID] = &edgetypes.EndpointUpdateScheduleRelation{
+			EnvironmentID: environmentID,
+			ScheduleID:    schedule.ID,
+			TargetVersion: environmentStatus.TargetVersion,
+			Status:        environmentStatus.Status,
+			Error:         environmentStatus.Error,
+			Type:          schedule.Type,
+		}
+	}
+
+	return nil
+}

api/dataservices/endpoint/endpoint.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -68,10 +69,12 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 		func(obj interface{}) (interface{}, error) {
 			endpoint, ok := obj.(*portainer.Endpoint)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Endpoint object")
-				return nil, fmt.Errorf("Failed to convert to Endpoint object: %s", obj)
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Endpoint object")
+				return nil, fmt.Errorf("failed to convert to Endpoint object: %s", obj)
 			}
+
 			endpoints = append(endpoints, *endpoint)
+
 			return &portainer.Endpoint{}, nil
 		})
 

api/dataservices/endpointgroup/endpointgroup.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -66,13 +67,14 @@ func (service *Service) EndpointGroups() ([]portainer.EndpointGroup, error) {
 		BucketName,
 		&portainer.EndpointGroup{},
 		func(obj interface{}) (interface{}, error) {
-			//var tag portainer.Tag
 			endpointGroup, ok := obj.(*portainer.EndpointGroup)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to EndpointGroup object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointGroup object")
 				return nil, fmt.Errorf("Failed to convert to EndpointGroup object: %s", obj)
 			}
+
 			endpointGroups = append(endpointGroups, *endpointGroup)
+
 			return &portainer.EndpointGroup{}, nil
 		})
 

api/dataservices/endpointrelation/endpointrelation.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -33,7 +34,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
-//EndpointRelations returns an array of all EndpointRelations
+// EndpointRelations returns an array of all EndpointRelations
 func (service *Service) EndpointRelations() ([]portainer.EndpointRelation, error) {
 	var all = make([]portainer.EndpointRelation, 0)
 
@@ -43,10 +44,12 @@ func (service *Service) EndpointRelations() ([]portainer.EndpointRelation, error
 		func(obj interface{}) (interface{}, error) {
 			r, ok := obj.(*portainer.EndpointRelation)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to EndpointRelation object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EndpointRelation object")
 				return nil, fmt.Errorf("Failed to convert to EndpointRelation object: %s", obj)
 			}
+
 			all = append(all, *r)
+
 			return &portainer.EndpointRelation{}, nil
 		})
 

api/dataservices/errors/errors.go

@@ -4,6 +4,7 @@ import "errors"
 
 var (
 	// TODO: i'm pretty sure this needs wrapping at several levels
-	ErrObjectNotFound = errors.New("Object not found inside the database")
-	ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
+	ErrObjectNotFound = errors.New("object not found inside the database")
+	ErrWrongDBEdition = errors.New("the Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
+	ErrDBImportFailed = errors.New("importing backup failed")
 )

api/dataservices/extension/extension.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -56,10 +57,12 @@ func (service *Service) Extensions() ([]portainer.Extension, error) {
 		func(obj interface{}) (interface{}, error) {
 			extension, ok := obj.(*portainer.Extension)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Extension object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Extension object")
 				return nil, fmt.Errorf("Failed to convert to Extension object: %s", obj)
 			}
+
 			extensions = append(extensions, *extension)
+
 			return &portainer.Extension{}, nil
 		})
 

api/dataservices/fdoprofile/fdoprofile.go

@@ -0,0 +1,95 @@
+package fdoprofile
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName = "fdo_profiles"
+)
+
+// Service represents a service for managingFDO Profiles data.
+type Service struct {
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
+}
+
+// NewService creates a new instance of a service.
+func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+// FDOProfiles return an array containing all the FDO Profiles.
+func (service *Service) FDOProfiles() ([]portainer.FDOProfile, error) {
+	var fdoProfiles = make([]portainer.FDOProfile, 0)
+
+	err := service.connection.GetAll(
+		BucketName,
+		&portainer.FDOProfile{},
+		func(obj interface{}) (interface{}, error) {
+			fdoProfile, ok := obj.(*portainer.FDOProfile)
+			if !ok {
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to FDOProfile object")
+
+				return nil, fmt.Errorf("Failed to convert to FDOProfile object: %s", obj)
+			}
+			fdoProfiles = append(fdoProfiles, *fdoProfile)
+			return &portainer.FDOProfile{}, nil
+		})
+
+	return fdoProfiles, err
+}
+
+// FDOProfile returns an FDO Profile by ID.
+func (service *Service) FDOProfile(ID portainer.FDOProfileID) (*portainer.FDOProfile, error) {
+	var FDOProfile portainer.FDOProfile
+	identifier := service.connection.ConvertToKey(int(ID))
+
+	err := service.connection.GetObject(BucketName, identifier, &FDOProfile)
+	if err != nil {
+		return nil, err
+	}
+
+	return &FDOProfile, nil
+}
+
+// Create assign an ID to a new FDO Profile and saves it.
+func (service *Service) Create(FDOProfile *portainer.FDOProfile) error {
+	return service.connection.CreateObjectWithId(
+		BucketName,
+		int(FDOProfile.ID),
+		FDOProfile,
+	)
+}
+
+// Update updates an FDO Profile.
+func (service *Service) Update(ID portainer.FDOProfileID, FDOProfile *portainer.FDOProfile) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.UpdateObject(BucketName, identifier, FDOProfile)
+}
+
+// Delete deletes an FDO Profile.
+func (service *Service) Delete(ID portainer.FDOProfileID) error {
+	identifier := service.connection.ConvertToKey(int(ID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}
+
+// GetNextIdentifier returns the next identifier for a FDO Profile.
+func (service *Service) GetNextIdentifier() int {
+	return service.connection.GetNextIdentifier(BucketName)
+}

api/dataservices/helmuserrepository/helmuserrepository.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -33,8 +34,8 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
-//HelmUserRepository returns an array of all HelmUserRepository
-func (service *Service) HelmUserRepositorys() ([]portainer.HelmUserRepository, error) {
+// HelmUserRepository returns an array of all HelmUserRepository
+func (service *Service) HelmUserRepositories() ([]portainer.HelmUserRepository, error) {
 	var repos = make([]portainer.HelmUserRepository, 0)
 
 	err := service.connection.GetAll(
@@ -43,10 +44,12 @@ func (service *Service) HelmUserRepositorys() ([]portainer.HelmUserRepository, e
 		func(obj interface{}) (interface{}, error) {
 			r, ok := obj.(*portainer.HelmUserRepository)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to HelmUserRepository object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
 				return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
 			}
+
 			repos = append(repos, *r)
+
 			return &portainer.HelmUserRepository{}, nil
 		})
 
@@ -63,12 +66,14 @@ func (service *Service) HelmUserRepositoryByUserID(userID portainer.UserID) ([]p
 		func(obj interface{}) (interface{}, error) {
 			record, ok := obj.(*portainer.HelmUserRepository)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to HelmUserRepository object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to HelmUserRepository object")
 				return nil, fmt.Errorf("Failed to convert to HelmUserRepository object: %s", obj)
 			}
+
 			if record.UserID == userID {
 				result = append(result, *record)
 			}
+
 			return &portainer.HelmUserRepository{}, nil
 		})
 

api/dataservices/interface.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/portainer/portainer/api/edgetypes"
 
 	portainer "github.com/portainer/portainer/api"
 )
@@ -28,15 +29,18 @@ type (
 		EdgeGroup() EdgeGroupService
 		EdgeJob() EdgeJobService
 		EdgeStack() EdgeStackService
+		EdgeUpdateSchedule() EdgeUpdateScheduleService
 		Endpoint() EndpointService
 		EndpointGroup() EndpointGroupService
 		EndpointRelation() EndpointRelationService
+		FDOProfile() FDOProfileService
 		HelmUserRepository() HelmUserRepositoryService
 		Registry() RegistryService
 		ResourceControl() ResourceControlService
 		Role() RoleService
 		APIKeyRepository() APIKeyRepository
 		Settings() SettingsService
+		Snapshot() SnapshotService
 		SSLSettings() SSLSettingsService
 		Stack() StackService
 		Tag() TagService
@@ -73,18 +77,29 @@ type (
 	EdgeJobService interface {
 		EdgeJobs() ([]portainer.EdgeJob, error)
 		EdgeJob(ID portainer.EdgeJobID) (*portainer.EdgeJob, error)
-		Create(edgeJob *portainer.EdgeJob) error
+		Create(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
 		UpdateEdgeJob(ID portainer.EdgeJobID, edgeJob *portainer.EdgeJob) error
 		DeleteEdgeJob(ID portainer.EdgeJobID) error
 		GetNextIdentifier() int
 		BucketName() string
 	}
 
+	EdgeUpdateScheduleService interface {
+		ActiveSchedule(environmentID portainer.EndpointID) *edgetypes.EndpointUpdateScheduleRelation
+		ActiveSchedules(environmentIDs []portainer.EndpointID) []edgetypes.EndpointUpdateScheduleRelation
+		List() ([]edgetypes.UpdateSchedule, error)
+		Item(ID edgetypes.UpdateScheduleID) (*edgetypes.UpdateSchedule, error)
+		Create(edgeUpdateSchedule *edgetypes.UpdateSchedule) error
+		Update(ID edgetypes.UpdateScheduleID, edgeUpdateSchedule *edgetypes.UpdateSchedule) error
+		Delete(ID edgetypes.UpdateScheduleID) error
+		BucketName() string
+	}
+
 	// EdgeStackService represents a service to manage Edge stacks
 	EdgeStackService interface {
 		EdgeStacks() ([]portainer.EdgeStack, error)
 		EdgeStack(ID portainer.EdgeStackID) (*portainer.EdgeStack, error)
-		Create(edgeStack *portainer.EdgeStack) error
+		Create(id portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error
 		UpdateEdgeStack(ID portainer.EdgeStackID, edgeStack *portainer.EdgeStack) error
 		DeleteEdgeStack(ID portainer.EdgeStackID) error
 		GetNextIdentifier() int
@@ -122,9 +137,20 @@ type (
 		BucketName() string
 	}
 
+	// FDOProfileService represents a service to manage FDO Profiles
+	FDOProfileService interface {
+		FDOProfiles() ([]portainer.FDOProfile, error)
+		FDOProfile(ID portainer.FDOProfileID) (*portainer.FDOProfile, error)
+		Create(FDOProfile *portainer.FDOProfile) error
+		Update(ID portainer.FDOProfileID, FDOProfile *portainer.FDOProfile) error
+		Delete(ID portainer.FDOProfileID) error
+		GetNextIdentifier() int
+		BucketName() string
+	}
+
 	// HelmUserRepositoryService represents a service to manage HelmUserRepositories
 	HelmUserRepositoryService interface {
-		HelmUserRepositorys() ([]portainer.HelmUserRepository, error)
+		HelmUserRepositories() ([]portainer.HelmUserRepository, error)
 		HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error)
 		Create(record *portainer.HelmUserRepository) error
 		UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, repository *portainer.HelmUserRepository) error
@@ -189,6 +215,15 @@ type (
 		BucketName() string
 	}
 
+	SnapshotService interface {
+		Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error)
+		Snapshots() ([]portainer.Snapshot, error)
+		UpdateSnapshot(snapshot *portainer.Snapshot) error
+		DeleteSnapshot(endpointID portainer.EndpointID) error
+		Create(snapshot *portainer.Snapshot) error
+		BucketName() string
+	}
+
 	// SSLSettingsService represents a service for managing application settings
 	SSLSettingsService interface {
 		Settings() (*portainer.SSLSettings, error)

api/dataservices/registry/registry.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -56,10 +57,12 @@ func (service *Service) Registries() ([]portainer.Registry, error) {
 		func(obj interface{}) (interface{}, error) {
 			registry, ok := obj.(*portainer.Registry)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Registry object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Registry object")
 				return nil, fmt.Errorf("Failed to convert to Registry object: %s", obj)
 			}
+
 			registries = append(registries, *registry)
+
 			return &portainer.Registry{}, nil
 		})
 

api/dataservices/resourcecontrol/resourcecontrol.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -59,7 +59,7 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 		func(obj interface{}) (interface{}, error) {
 			rc, ok := obj.(*portainer.ResourceControl)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to ResourceControl object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
 				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
 			}
 
@@ -74,14 +74,12 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
 					return nil, stop
 				}
 			}
+
 			return &portainer.ResourceControl{}, nil
 		})
 	if err == stop {
 		return resourceControl, nil
 	}
-	if err == nil {
-		return nil, errors.ErrObjectNotFound
-	}
 
 	return nil, err
 }
@@ -96,10 +94,12 @@ func (service *Service) ResourceControls() ([]portainer.ResourceControl, error)
 		func(obj interface{}) (interface{}, error) {
 			rc, ok := obj.(*portainer.ResourceControl)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to ResourceControl object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
 				return nil, fmt.Errorf("Failed to convert to ResourceControl object: %s", obj)
 			}
+
 			rcs = append(rcs, *rc)
+
 			return &portainer.ResourceControl{}, nil
 		})
 

api/dataservices/role/role.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -56,10 +57,12 @@ func (service *Service) Roles() ([]portainer.Role, error) {
 		func(obj interface{}) (interface{}, error) {
 			set, ok := obj.(*portainer.Role)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Role object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Role object")
 				return nil, fmt.Errorf("Failed to convert to Role object: %s", obj)
 			}
+
 			sets = append(sets, *set)
+
 			return &portainer.Role{}, nil
 		})
 

api/dataservices/schedule/schedule.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -68,10 +69,12 @@ func (service *Service) Schedules() ([]portainer.Schedule, error) {
 		func(obj interface{}) (interface{}, error) {
 			schedule, ok := obj.(*portainer.Schedule)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Schedule object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
 				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
 			}
+
 			schedules = append(schedules, *schedule)
+
 			return &portainer.Schedule{}, nil
 		})
 
@@ -89,12 +92,14 @@ func (service *Service) SchedulesByJobType(jobType portainer.JobType) ([]portain
 		func(obj interface{}) (interface{}, error) {
 			schedule, ok := obj.(*portainer.Schedule)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Schedule object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Schedule object")
 				return nil, fmt.Errorf("Failed to convert to Schedule object: %s", obj)
 			}
+
 			if schedule.JobType == jobType {
 				schedules = append(schedules, *schedule)
 			}
+
 			return &portainer.Schedule{}, nil
 		})
 

api/dataservices/snapshot/snapshot.go

@@ -0,0 +1,76 @@
+package snapshot
+
+import (
+	"fmt"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	BucketName = "snapshots"
+)
+
+type Service struct {
+	connection portainer.Connection
+}
+
+func (service *Service) BucketName() string {
+	return BucketName
+}
+
+func NewService(connection portainer.Connection) (*Service, error) {
+	err := connection.SetServiceName(BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		connection: connection,
+	}, nil
+}
+
+func (service *Service) Snapshot(endpointID portainer.EndpointID) (*portainer.Snapshot, error) {
+	var snapshot portainer.Snapshot
+	identifier := service.connection.ConvertToKey(int(endpointID))
+
+	err := service.connection.GetObject(BucketName, identifier, &snapshot)
+	if err != nil {
+		return nil, err
+	}
+
+	return &snapshot, nil
+}
+
+func (service *Service) Snapshots() ([]portainer.Snapshot, error) {
+	var snapshots = make([]portainer.Snapshot, 0)
+
+	err := service.connection.GetAllWithJsoniter(
+		BucketName,
+		&portainer.Snapshot{},
+		func(obj interface{}) (interface{}, error) {
+			snapshot, ok := obj.(*portainer.Snapshot)
+			if !ok {
+				logrus.WithField("obj", obj).Errorf("Failed to convert to Snapshot object")
+				return nil, fmt.Errorf("failed to convert to Snapshot object: %s", obj)
+			}
+			snapshots = append(snapshots, *snapshot)
+			return &portainer.Snapshot{}, nil
+		})
+
+	return snapshots, err
+}
+
+func (service *Service) UpdateSnapshot(snapshot *portainer.Snapshot) error {
+	identifier := service.connection.ConvertToKey(int(snapshot.EndpointID))
+	return service.connection.UpdateObject(BucketName, identifier, snapshot)
+}
+
+func (service *Service) DeleteSnapshot(endpointID portainer.EndpointID) error {
+	identifier := service.connection.ConvertToKey(int(endpointID))
+	return service.connection.DeleteObject(BucketName, identifier)
+}
+
+func (service *Service) Create(snapshot *portainer.Snapshot) error {
+	return service.connection.CreateObjectWithId(BucketName, int(snapshot.EndpointID), snapshot)
+}

api/dataservices/stack/stack.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/sirupsen/logrus"
-
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -60,13 +60,15 @@ func (service *Service) StackByName(name string) (*portainer.Stack, error) {
 		func(obj interface{}) (interface{}, error) {
 			stack, ok := obj.(*portainer.Stack)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Stack object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
 				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
 			}
+
 			if stack.Name == name {
 				s = stack
 				return nil, stop
 			}
+
 			return &portainer.Stack{}, nil
 		})
 	if err == stop {
@@ -89,12 +91,14 @@ func (service *Service) StacksByName(name string) ([]portainer.Stack, error) {
 		func(obj interface{}) (interface{}, error) {
 			stack, ok := obj.(portainer.Stack)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Stack object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
 				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
 			}
+
 			if stack.Name == name {
 				stacks = append(stacks, stack)
 			}
+
 			return &portainer.Stack{}, nil
 		})
 
@@ -111,10 +115,12 @@ func (service *Service) Stacks() ([]portainer.Stack, error) {
 		func(obj interface{}) (interface{}, error) {
 			stack, ok := obj.(*portainer.Stack)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Stack object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
 				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
 			}
+
 			stacks = append(stacks, *stack)
+
 			return &portainer.Stack{}, nil
 		})
 
@@ -156,13 +162,15 @@ func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) {
 			s, ok = obj.(*portainer.Stack)
 
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Stack object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
+
 				return &portainer.Stack{}, nil
 			}
 
 			if s.AutoUpdate != nil && strings.EqualFold(s.AutoUpdate.Webhook, id) {
 				return nil, stop
 			}
+
 			return &portainer.Stack{}, nil
 		})
 	if err == stop {
@@ -186,12 +194,14 @@ func (service *Service) RefreshableStacks() ([]portainer.Stack, error) {
 		func(obj interface{}) (interface{}, error) {
 			stack, ok := obj.(*portainer.Stack)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Stack object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Stack object")
 				return nil, fmt.Errorf("Failed to convert to Stack object: %s", obj)
 			}
+
 			if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" {
 				stacks = append(stacks, *stack)
 			}
+
 			return &portainer.Stack{}, nil
 		})
 

api/dataservices/stack/tests/stack_test.go

@@ -29,7 +29,7 @@ func TestService_StackByWebhookID(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	b := stackBuilder{t: t, store: store}
@@ -87,7 +87,7 @@ func Test_RefreshableStacks(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test in short mode. Normally takes ~1s to run.")
 	}
-	_, store, teardown := datastore.MustNewTestStore(true)
+	_, store, teardown := datastore.MustNewTestStore(t, true, true)
 	defer teardown()
 
 	staticStack := portainer.Stack{ID: 1}

api/dataservices/tag/tag.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -43,10 +44,12 @@ func (service *Service) Tags() ([]portainer.Tag, error) {
 		func(obj interface{}) (interface{}, error) {
 			tag, ok := obj.(*portainer.Tag)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Tag object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Tag object")
 				return nil, fmt.Errorf("Failed to convert to Tag object: %s", obj)
 			}
+
 			tags = append(tags, *tag)
+
 			return &portainer.Tag{}, nil
 		})
 

api/dataservices/team/team.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/sirupsen/logrus"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -60,13 +60,15 @@ func (service *Service) TeamByName(name string) (*portainer.Team, error) {
 		func(obj interface{}) (interface{}, error) {
 			team, ok := obj.(*portainer.Team)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Team object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Team object")
 				return nil, fmt.Errorf("Failed to convert to Team object: %s", obj)
 			}
-			if strings.EqualFold(t.Name, name) {
+
+			if strings.EqualFold(team.Name, name) {
 				t = team
 				return nil, stop
 			}
+
 			return &portainer.Team{}, nil
 		})
 	if err == stop {
@@ -89,10 +91,12 @@ func (service *Service) Teams() ([]portainer.Team, error) {
 		func(obj interface{}) (interface{}, error) {
 			team, ok := obj.(*portainer.Team)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Team object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Team object")
 				return nil, fmt.Errorf("Failed to convert to Team object: %s", obj)
 			}
+
 			teams = append(teams, *team)
+
 			return &portainer.Team{}, nil
 		})
 

api/dataservices/team/tests/team_test.go

@@ -0,0 +1,53 @@
+package tests
+
+import (
+	"testing"
+
+	"github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/portainer/portainer/api/datastore"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_teamByName(t *testing.T) {
+	t.Run("When store is empty should return ErrObjectNotFound", func(t *testing.T) {
+		_, store, teardown := datastore.MustNewTestStore(t, true, true)
+		defer teardown()
+
+		_, err := store.Team().TeamByName("name")
+		assert.ErrorIs(t, err, errors.ErrObjectNotFound)
+
+	})
+
+	t.Run("When there is no object with the same name should return ErrObjectNotFound", func(t *testing.T) {
+		_, store, teardown := datastore.MustNewTestStore(t, true, true)
+		defer teardown()
+
+		teamBuilder := teamBuilder{
+			t:     t,
+			store: store,
+			count: 0,
+		}
+
+		teamBuilder.createNew("name1")
+
+		_, err := store.Team().TeamByName("name")
+		assert.ErrorIs(t, err, errors.ErrObjectNotFound)
+	})
+
+	t.Run("When there is an object with the same name should return the object", func(t *testing.T) {
+		_, store, teardown := datastore.MustNewTestStore(t, true, true)
+		defer teardown()
+
+		teamBuilder := teamBuilder{
+			t:     t,
+			store: store,
+			count: 0,
+		}
+
+		expectedTeam := teamBuilder.createNew("name1")
+
+		team, err := store.Team().TeamByName("name1")
+		assert.NoError(t, err, "TeamByName should succeed")
+		assert.Equal(t, expectedTeam, team)
+	})
+}

api/dataservices/team/tests/utils.go

@@ -0,0 +1,28 @@
+package tests
+
+import (
+	"testing"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore"
+	"github.com/stretchr/testify/assert"
+)
+
+type teamBuilder struct {
+	t     *testing.T
+	count int
+	store *datastore.Store
+}
+
+func (b *teamBuilder) createNew(name string) *portainer.Team {
+	b.count++
+	team := &portainer.Team{
+		ID:   portainer.TeamID(b.count),
+		Name: name,
+	}
+
+	err := b.store.Team().Create(team)
+	assert.NoError(b.t, err)
+
+	return team
+}

api/dataservices/teammembership/teammembership.go

@@ -4,7 +4,8 @@ import (
 	"fmt"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -56,10 +57,12 @@ func (service *Service) TeamMemberships() ([]portainer.TeamMembership, error) {
 		func(obj interface{}) (interface{}, error) {
 			membership, ok := obj.(*portainer.TeamMembership)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to TeamMembership object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
 				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
 			}
+
 			memberships = append(memberships, *membership)
+
 			return &portainer.TeamMembership{}, nil
 		})
 
@@ -76,12 +79,14 @@ func (service *Service) TeamMembershipsByUserID(userID portainer.UserID) ([]port
 		func(obj interface{}) (interface{}, error) {
 			membership, ok := obj.(*portainer.TeamMembership)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to TeamMembership object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
 				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
 			}
+
 			if membership.UserID == userID {
 				memberships = append(memberships, *membership)
 			}
+
 			return &portainer.TeamMembership{}, nil
 		})
 
@@ -98,12 +103,14 @@ func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]port
 		func(obj interface{}) (interface{}, error) {
 			membership, ok := obj.(*portainer.TeamMembership)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to TeamMembership object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
 				return nil, fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
 			}
+
 			if membership.TeamID == teamID {
 				memberships = append(memberships, *membership)
 			}
+
 			return &portainer.TeamMembership{}, nil
 		})
 
@@ -140,13 +147,15 @@ func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) er
 		func(obj interface{}) (id int, ok bool) {
 			membership, ok := obj.(portainer.TeamMembership)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to TeamMembership object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
 				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
 				return -1, false
 			}
+
 			if membership.UserID == userID {
 				return int(membership.ID), true
 			}
+
 			return -1, false
 		})
 }
@@ -158,13 +167,15 @@ func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) er
 		func(obj interface{}) (id int, ok bool) {
 			membership, ok := obj.(portainer.TeamMembership)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to TeamMembership object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
 				//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
 				return -1, false
 			}
+
 			if membership.TeamID == teamID {
 				return int(membership.ID), true
 			}
+
 			return -1, false
 		})
 }

api/dataservices/user/user.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/sirupsen/logrus"
-
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -59,18 +59,23 @@ func (service *Service) UserByUsername(username string) (*portainer.User, error)
 		func(obj interface{}) (interface{}, error) {
 			user, ok := obj.(*portainer.User)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to User object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
+
 				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
 			}
+
 			if strings.EqualFold(user.Username, username) {
 				u = user
 				return nil, stop
 			}
+
 			return &portainer.User{}, nil
 		})
+
 	if err == stop {
 		return u, nil
 	}
+
 	if err == nil {
 		return nil, errors.ErrObjectNotFound
 	}
@@ -88,10 +93,13 @@ func (service *Service) Users() ([]portainer.User, error) {
 		func(obj interface{}) (interface{}, error) {
 			user, ok := obj.(*portainer.User)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to User object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
+
 				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
 			}
+
 			users = append(users, *user)
+
 			return &portainer.User{}, nil
 		})
 
@@ -108,12 +116,15 @@ func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User,
 		func(obj interface{}) (interface{}, error) {
 			user, ok := obj.(*portainer.User)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to User object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to User object")
+
 				return nil, fmt.Errorf("Failed to convert to User object: %s", obj)
 			}
+
 			if user.Role == role {
 				users = append(users, *user)
 			}
+
 			return &portainer.User{}, nil
 		})
 

api/dataservices/webhook/webhook.go

@@ -5,7 +5,8 @@ import (
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices/errors"
-	"github.com/sirupsen/logrus"
+
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -34,7 +35,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
 	}, nil
 }
 
-//Webhooks returns an array of all webhooks
+// Webhooks returns an array of all webhooks
 func (service *Service) Webhooks() ([]portainer.Webhook, error) {
 	var webhooks = make([]portainer.Webhook, 0)
 
@@ -44,10 +45,12 @@ func (service *Service) Webhooks() ([]portainer.Webhook, error) {
 		func(obj interface{}) (interface{}, error) {
 			webhook, ok := obj.(*portainer.Webhook)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Webhook object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
 				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
 			}
+
 			webhooks = append(webhooks, *webhook)
+
 			return &portainer.Webhook{}, nil
 		})
 
@@ -77,18 +80,23 @@ func (service *Service) WebhookByResourceID(ID string) (*portainer.Webhook, erro
 		func(obj interface{}) (interface{}, error) {
 			webhook, ok := obj.(*portainer.Webhook)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Webhook object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+
 				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
 			}
+
 			if webhook.ResourceID == ID {
 				w = webhook
 				return nil, stop
 			}
+
 			return &portainer.Webhook{}, nil
 		})
+
 	if err == stop {
 		return w, nil
 	}
+
 	if err == nil {
 		return nil, errors.ErrObjectNotFound
 	}
@@ -106,18 +114,23 @@ func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error)
 		func(obj interface{}) (interface{}, error) {
 			webhook, ok := obj.(*portainer.Webhook)
 			if !ok {
-				logrus.WithField("obj", obj).Errorf("Failed to convert to Webhook object")
+				log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to Webhook object")
+
 				return nil, fmt.Errorf("Failed to convert to Webhook object: %s", obj)
 			}
+
 			if webhook.Token == token {
 				w = webhook
 				return nil, stop
 			}
+
 			return &portainer.Webhook{}, nil
 		})
+
 	if err == stop {
 		return w, nil
 	}
+
 	if err == nil {
 		return nil, errors.ErrObjectNotFound
 	}

api/datastore/backup.go

@@ -6,7 +6,7 @@ import (
 	"path"
 	"time"
 
-	plog "github.com/portainer/portainer/api/datastore/log"
+	"github.com/rs/zerolog/log"
 )
 
 var backupDefaults = struct {
@@ -17,8 +17,6 @@ var backupDefaults = struct {
 	"common",
 }
 
-var backupLog = plog.NewScopedLog("database, backup")
-
 //
 // Backup Helpers
 //
@@ -29,13 +27,13 @@ func (store *Store) createBackupFolders() {
 	commonDir := store.commonBackupDir()
 	if exists, _ := store.fileService.FileExists(commonDir); !exists {
 		if err := os.MkdirAll(commonDir, 0700); err != nil {
-			backupLog.Error("Error while creating common backup folder", err)
+			log.Error().Err(err).Msg("error while creating common backup folder")
 		}
 	}
 }
 
 func (store *Store) databasePath() string {
-	return path.Join(store.connection.GetStorePath(), store.connection.GetDatabaseFilename())
+	return store.connection.GetDatabaseFilePath()
 }
 
 func (store *Store) commonBackupDir() string {
@@ -43,11 +41,13 @@ func (store *Store) commonBackupDir() string {
 }
 
 func (store *Store) copyDBFile(from string, to string) error {
-	backupLog.Info(fmt.Sprintf("Copying db file from %s to %s", from, to))
+	log.Info().Str("from", from).Str("to", to).Msg("copying DB file")
+
 	err := store.fileService.Copy(from, to, true)
 	if err != nil {
-		backupLog.Error("Failed", err)
+		log.Error().Err(err).Msg("failed")
 	}
+
 	return err
 }
 
@@ -69,6 +69,11 @@ func getBackupRestoreOptions(backupDir string) *BackupOptions {
 	}
 }
 
+// Backup current database with default options
+func (store *Store) Backup() (string, error) {
+	return store.backupWithOptions(nil)
+}
+
 func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
 	if options == nil {
 		options = &BackupOptions{}
@@ -84,7 +89,7 @@ func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
 		options.BackupDir = store.commonBackupDir()
 	}
 	if options.BackupFileName == "" {
-		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFilename(), fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
+		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
 	}
 	if options.BackupPath == "" {
 		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
@@ -94,12 +99,32 @@ func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
 
 // BackupWithOptions backup current database with options
 func (store *Store) backupWithOptions(options *BackupOptions) (string, error) {
-	backupLog.Info("creating db backup")
+	log.Info().Msg("creating DB backup")
+
 	store.createBackupFolders()
 
 	options = store.setupOptions(options)
+	dbPath := store.databasePath()
 
-	return options.BackupPath, store.copyDBFile(store.databasePath(), options.BackupPath)
+	if err := store.Close(); err != nil {
+		return options.BackupPath, fmt.Errorf(
+			"error closing datastore before creating backup: %v",
+			err,
+		)
+	}
+
+	if err := store.copyDBFile(dbPath, options.BackupPath); err != nil {
+		return options.BackupPath, err
+	}
+
+	if _, err := store.Open(); err != nil {
+		return options.BackupPath, fmt.Errorf(
+			"error opening datastore after creating backup: %v",
+			err,
+		)
+	}
+
+	return options.BackupPath, nil
 }
 
 // RestoreWithOptions previously saved backup for the current Edition  with options
@@ -112,17 +137,19 @@ func (store *Store) restoreWithOptions(options *BackupOptions) error {
 	// Check if backup file exist before restoring
 	_, err := os.Stat(options.BackupPath)
 	if os.IsNotExist(err) {
-		backupLog.Error(fmt.Sprintf("Backup file to restore does not exist %s", options.BackupPath), err)
+		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to restore does not exist %s")
+
 		return err
 	}
 
 	err = store.Close()
 	if err != nil {
-		backupLog.Error("Error while closing store before restore", err)
+		log.Error().Err(err).Msg("error while closing store before restore")
+
 		return err
 	}
 
-	backupLog.Info("Restoring db backup")
+	log.Info().Msg("restoring DB backup")
 	err = store.copyDBFile(options.BackupPath, store.databasePath())
 	if err != nil {
 		return err
@@ -134,20 +161,22 @@ func (store *Store) restoreWithOptions(options *BackupOptions) error {
 
 // RemoveWithOptions removes backup database based on supplied options
 func (store *Store) removeWithOptions(options *BackupOptions) error {
-	backupLog.Info("Removing db backup")
+	log.Info().Msg("removing DB backup")
 
 	options = store.setupOptions(options)
 	_, err := os.Stat(options.BackupPath)
 
 	if os.IsNotExist(err) {
-		backupLog.Error(fmt.Sprintf("Backup file to remove does not exist %s", options.BackupPath), err)
+		log.Error().Str("path", options.BackupPath).Err(err).Msg("backup file to remove does not exist")
+
 		return err
 	}
 
-	backupLog.Info(fmt.Sprintf("Removing db file at %s", options.BackupPath))
+	log.Info().Str("path", options.BackupPath).Msg("removing DB file")
 	err = os.Remove(options.BackupPath)
 	if err != nil {
-		backupLog.Error("Failed", err)
+		log.Error().Err(err).Msg("failed")
+
 		return err
 	}
 

api/datastore/backup_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestCreateBackupFolders(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false)
+	_, store, teardown := MustNewTestStore(t, false, true)
 	defer teardown()
 
 	connection := store.GetConnection()
@@ -27,7 +27,7 @@ func TestCreateBackupFolders(t *testing.T) {
 }
 
 func TestStoreCreation(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true)
+	_, store, teardown := MustNewTestStore(t, true, true)
 	defer teardown()
 
 	if store == nil {
@@ -40,7 +40,7 @@ func TestStoreCreation(t *testing.T) {
 }
 
 func TestBackup(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true)
+	_, store, teardown := MustNewTestStore(t, true, true)
 	connection := store.GetConnection()
 	defer teardown()
 
@@ -48,7 +48,7 @@ func TestBackup(t *testing.T) {
 		store.VersionService.StoreDBVersion(portainer.DBVersion)
 		store.backupWithOptions(nil)
 
-		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.db.%03d.*", portainer.DBVersion))
+		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%03d.*", portainer.DBVersion))
 		if !isFileExist(backupFileName) {
 			t.Errorf("Expect backup file to be created %s", backupFileName)
 		}
@@ -67,7 +67,7 @@ func TestBackup(t *testing.T) {
 }
 
 func TestRemoveWithOptions(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true)
+	_, store, teardown := MustNewTestStore(t, true, true)
 	defer teardown()
 
 	t.Run("successfully removes file if existent", func(t *testing.T) {
@@ -86,7 +86,7 @@ func TestRemoveWithOptions(t *testing.T) {
 
 		err = store.removeWithOptions(options)
 		if err != nil {
-			t.Errorf("RemoveWithOptions should successfully remove file; err=%w", err)
+			t.Errorf("RemoveWithOptions should successfully remove file; err=%v", err)
 		}
 
 		if isFileExist(f.Name()) {

api/datastore/datastore.go

@@ -1,10 +1,16 @@
 package datastore
 
 import (
+	"fmt"
 	"io"
+	"os"
+	"path"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
+	portainerErrors "github.com/portainer/portainer/api/dataservices/errors"
+
+	"github.com/rs/zerolog/log"
 )
 
 func (store *Store) version() (int, error) {
@@ -34,6 +40,19 @@ func NewStore(storePath string, fileService portainer.FileService, connection po
 // Open opens and initializes the BoltDB database.
 func (store *Store) Open() (newStore bool, err error) {
 	newStore = true
+
+	encryptionReq, err := store.connection.NeedsEncryptionMigration()
+	if err != nil {
+		return false, err
+	}
+
+	if encryptionReq {
+		err = store.encryptDB()
+		if err != nil {
+			return false, err
+		}
+	}
+
 	err = store.connection.Open()
 	if err != nil {
 		return newStore, err
@@ -45,8 +64,19 @@ func (store *Store) Open() (newStore bool, err error) {
 	}
 
 	// if we have DBVersion in the database then ensure we flag this as NOT a new store
-	if _, err := store.VersionService.DBVersion(); err == nil {
-		newStore = false
+	version, err := store.VersionService.DBVersion()
+	if err != nil {
+		if store.IsErrObjectNotFound(err) {
+			return newStore, nil
+		}
+
+		return newStore, err
+	}
+
+	if version > 0 {
+		log.Debug().Int("version", version).Msg("opened existing store")
+
+		return false, nil
 	}
 
 	return newStore, nil
@@ -65,16 +95,84 @@ func (store *Store) BackupTo(w io.Writer) error {
 // CheckCurrentEdition checks if current edition is community edition
 func (store *Store) CheckCurrentEdition() error {
 	if store.edition() != portainer.PortainerCE {
-		return errors.ErrWrongDBEdition
+		return portainerErrors.ErrWrongDBEdition
 	}
 	return nil
 }
 
 // TODO: move the use of this to dataservices.IsErrObjectNotFound()?
 func (store *Store) IsErrObjectNotFound(e error) bool {
-	return e == errors.ErrObjectNotFound
+	return e == portainerErrors.ErrObjectNotFound
 }
 
 func (store *Store) Rollback(force bool) error {
 	return store.connectionRollback(force)
 }
+
+func (store *Store) encryptDB() error {
+	store.connection.SetEncrypted(false)
+	err := store.connection.Open()
+	if err != nil {
+		return err
+	}
+
+	err = store.initServices()
+	if err != nil {
+		return err
+	}
+
+	// The DB is not currently encrypted.  First save the encrypted db filename
+	oldFilename := store.connection.GetDatabaseFilePath()
+	log.Info().Msg("encrypting database")
+
+	// export file path for backup
+	exportFilename := path.Join(store.databasePath() + "." + fmt.Sprintf("backup-%d.json", time.Now().Unix()))
+
+	log.Info().Str("filename", exportFilename).Msg("exporting database backup")
+
+	err = store.Export(exportFilename)
+	if err != nil {
+		log.Error().Str("filename", exportFilename).Err(err).Msg("failed to export")
+
+		return err
+	}
+
+	log.Info().Msg("database backup exported")
+
+	// Close existing un-encrypted db so that we can delete the file later
+	store.connection.Close()
+
+	// Tell the db layer to create an encrypted db when opened
+	store.connection.SetEncrypted(true)
+	store.connection.Open()
+
+	// We have to init services before import
+	err = store.initServices()
+	if err != nil {
+		return err
+	}
+
+	err = store.Import(exportFilename)
+	if err != nil {
+		// Remove the new encrypted file that we failed to import
+		os.Remove(store.connection.GetDatabaseFilePath())
+		log.Fatal().Err(portainerErrors.ErrDBImportFailed).Msg("")
+	}
+
+	err = os.Remove(oldFilename)
+	if err != nil {
+		log.Error().Msg("failed to remove the un-encrypted db file")
+	}
+
+	err = os.Remove(exportFilename)
+	if err != nil {
+		log.Error().Msg("failed to remove the json backup file")
+	}
+
+	// Close db connection
+	store.connection.Close()
+
+	log.Info().Msg("database successfully encrypted")
+
+	return nil
+}

api/datastore/datastore_test.go

@@ -27,7 +27,7 @@ const (
 // TestStoreFull an eventually comprehensive set of tests for the Store.
 // The idea is what we write to the store, we should read back.
 func TestStoreFull(t *testing.T) {
-	_, store, teardown := MustNewTestStore(true)
+	_, store, teardown := MustNewTestStore(t, true, true)
 	defer teardown()
 
 	testCases := map[string]func(t *testing.T){
@@ -96,7 +96,6 @@ func newEndpoint(endpointType portainer.EndpointType, id portainer.EndpointID, n
 		},
 		UserAccessPolicies: portainer.UserAccessPolicies{},
 		TeamAccessPolicies: portainer.TeamAccessPolicies{},
-		Extensions:         []portainer.EndpointExtension{},
 		TagIDs:             []portainer.TagID{},
 		Status:             portainer.EndpointStatusUp,
 		Snapshots:          []portainer.DockerSnapshot{},

api/datastore/init.go

@@ -7,41 +7,64 @@ import (
 
 // Init creates the default data set.
 func (store *Store) Init() error {
-	instanceID, err := store.VersionService.InstanceID()
+	err := store.checkOrCreateInstanceID()
+	if err != nil {
+		return err
+	}
+
+	err = store.checkOrCreateDefaultSettings()
+	if err != nil {
+		return err
+	}
+
+	err = store.checkOrCreateDefaultSSLSettings()
+	if err != nil {
+		return err
+	}
+
+	return store.checkOrCreateDefaultData()
+}
+
+func (store *Store) checkOrCreateInstanceID() error {
+	_, err := store.VersionService.InstanceID()
 	if store.IsErrObjectNotFound(err) {
 		uid, err := uuid.NewV4()
 		if err != nil {
 			return err
 		}
 
-		instanceID = uid.String()
-		err = store.VersionService.StoreInstanceID(instanceID)
-		if err != nil {
-			return err
-		}
-	} else if err != nil {
-		return err
+		instanceID := uid.String()
+		return store.VersionService.StoreInstanceID(instanceID)
 	}
+	return err
+}
 
+func (store *Store) checkOrCreateDefaultSettings() error {
 	// TODO: these need to also be applied when importing
 	settings, err := store.SettingsService.Settings()
 	if store.IsErrObjectNotFound(err) {
 		defaultSettings := &portainer.Settings{
+			EnableTelemetry:      false,
 			AuthenticationMethod: portainer.AuthenticationInternal,
 			BlackListedLabels:    make([]portainer.Pair, 0),
+			InternalAuthSettings: portainer.InternalAuthSettings{
+				RequiredPasswordLength: 12,
+			},
 			LDAPSettings: portainer.LDAPSettings{
 				AnonymousMode:   true,
 				AutoCreateUsers: true,
 				TLSConfig:       portainer.TLSConfiguration{},
 				SearchSettings: []portainer.LDAPSearchSettings{
-					portainer.LDAPSearchSettings{},
+					{},
 				},
 				GroupSearchSettings: []portainer.LDAPGroupSearchSettings{
-					portainer.LDAPGroupSearchSettings{},
+					{},
 				},
 			},
-			OAuthSettings: portainer.OAuthSettings{},
-
+			OAuthSettings: portainer.OAuthSettings{
+				SSO: true,
+			},
+			SnapshotInterval:         portainer.DefaultSnapshotInterval,
 			EdgeAgentCheckinInterval: portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
 			TemplatesURL:             portainer.DefaultTemplatesURL,
 			HelmRepositoryURL:        portainer.DefaultHelmRepositoryURL,
@@ -50,35 +73,33 @@ func (store *Store) Init() error {
 			KubectlShellImage:        portainer.DefaultKubectlShellImage,
 		}
 
-		err = store.SettingsService.UpdateSettings(defaultSettings)
-		if err != nil {
-			return err
-		}
-	} else if err != nil {
+		return store.SettingsService.UpdateSettings(defaultSettings)
+	}
+	if err != nil {
 		return err
-	} else if err == nil {
-		if settings.UserSessionTimeout == "" {
-			settings.UserSessionTimeout = portainer.DefaultUserSessionTimeout
-			store.Settings().UpdateSettings(settings)
-		}
 	}
 
-	_, err = store.SSLSettings().Settings()
-	if err != nil {
-		if !store.IsErrObjectNotFound(err) {
-			return err
-		}
+	if settings.UserSessionTimeout == "" {
+		settings.UserSessionTimeout = portainer.DefaultUserSessionTimeout
+		return store.Settings().UpdateSettings(settings)
+	}
+	return nil
+}
 
+func (store *Store) checkOrCreateDefaultSSLSettings() error {
+	_, err := store.SSLSettings().Settings()
+
+	if store.IsErrObjectNotFound(err) {
 		defaultSSLSettings := &portainer.SSLSettings{
 			HTTPEnabled: true,
 		}
 
-		err = store.SSLSettings().UpdateSettings(defaultSSLSettings)
-		if err != nil {
-			return err
-		}
+		return store.SSLSettings().UpdateSettings(defaultSSLSettings)
 	}
+	return err
+}
 
+func (store *Store) checkOrCreateDefaultData() error {
 	groups, err := store.EndpointGroupService.EndpointGroups()
 	if err != nil {
 		return err
@@ -99,6 +120,5 @@ func (store *Store) Init() error {
 			return err
 		}
 	}
-
 	return nil
 }

api/datastore/log/log.go

@@ -1,41 +0,0 @@
-package log
-
-import (
-	"fmt"
-	"log"
-)
-
-const (
-	INFO  = "INFO"
-	ERROR = "ERROR"
-	DEBUG = "DEBUG"
-	FATAL = "FATAL"
-)
-
-type ScopedLog struct {
-	scope string
-}
-
-func NewScopedLog(scope string) *ScopedLog {
-	return &ScopedLog{scope: scope}
-}
-
-func (slog *ScopedLog) print(kind string, message string) {
-	log.Printf("[%s] [%s] %s", kind, slog.scope, message)
-}
-
-func (slog *ScopedLog) Debug(message string) {
-	slog.print(DEBUG, fmt.Sprintf("[message: %s]", message))
-}
-
-func (slog *ScopedLog) Info(message string) {
-	slog.print(INFO, fmt.Sprintf("[message: %s]", message))
-}
-
-func (slog *ScopedLog) Error(message string, err error) {
-	slog.print(ERROR, fmt.Sprintf("[message: %s] [error: %s]", message, err))
-}
-
-func (slog *ScopedLog) NotImplemented(method string) {
-	log.Fatalf("[%s] [%s] [%s]", FATAL, slog.scope, fmt.Sprintf("%s is not yet implemented", method))
-}

api/datastore/migrate_data.go

@@ -4,37 +4,43 @@ import (
 	"fmt"
 	"runtime/debug"
 
+	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/cli"
 	"github.com/portainer/portainer/api/dataservices/errors"
-	plog "github.com/portainer/portainer/api/datastore/log"
 	"github.com/portainer/portainer/api/datastore/migrator"
 	"github.com/portainer/portainer/api/internal/authorization"
 
 	werrors "github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/rs/zerolog/log"
 )
 
 const beforePortainerVersionUpgradeBackup = "portainer.db.bak"
 
-var migrateLog = plog.NewScopedLog("database, migrate")
-
 func (store *Store) MigrateData() error {
 	version, err := store.version()
 	if err != nil {
 		return err
 	}
 
+	// Backup Database
+	backupPath, err := store.Backup()
+	if err != nil {
+		return werrors.Wrap(err, "while backing up db before migration")
+	}
+
 	migratorParams := &migrator.MigratorParameters{
 		DatabaseVersion:         version,
 		EndpointGroupService:    store.EndpointGroupService,
 		EndpointService:         store.EndpointService,
 		EndpointRelationService: store.EndpointRelationService,
 		ExtensionService:        store.ExtensionService,
+		FDOProfilesService:      store.FDOProfilesService,
 		RegistryService:         store.RegistryService,
 		ResourceControlService:  store.ResourceControlService,
 		RoleService:             store.RoleService,
 		ScheduleService:         store.ScheduleService,
 		SettingsService:         store.SettingsService,
+		SnapshotService:         store.SnapshotService,
 		StackService:            store.StackService,
 		TagService:              store.TagService,
 		TeamMembershipService:   store.TeamMembershipService,
@@ -45,7 +51,26 @@ func (store *Store) MigrateData() error {
 		AuthorizationService:    authorization.NewService(store),
 	}
 
-	return store.connectionMigrateData(migratorParams)
+	// restore on error
+	err = store.connectionMigrateData(migratorParams)
+	if err != nil {
+		log.Error().Err(err).Msg("while DB migration, restoring DB")
+
+		// Restore options
+		options := BackupOptions{
+			BackupPath: backupPath,
+		}
+
+		err := store.restoreWithOptions(&options)
+		if err != nil {
+			log.Fatal().
+				Str("database_file", store.databasePath()).
+				Str("backup", options.BackupPath).Err(err).
+				Msg("failed restoring the backup, Portainer database file needs to restored manually by replacing the database file with a recent backup")
+		}
+	}
+
+	return err
 }
 
 // FailSafeMigrate backup and restore DB if migration fail
@@ -83,10 +108,15 @@ func (store *Store) connectionMigrateData(migratorParams *migrator.MigratorParam
 	}
 
 	if migrator.Version() < portainer.DBVersion {
-		migrateLog.Info(fmt.Sprintf("Migrating database from version %v to %v.\n", migrator.Version(), portainer.DBVersion))
+		log.Info().
+			Int("migrator_version", migrator.Version()).
+			Int("db_version", portainer.DBVersion).
+			Msg("migrating database")
+
 		err = store.FailSafeMigrate(migrator)
 		if err != nil {
-			migrateLog.Error("An error occurred during database migration", err)
+			log.Error().Err(err).Msg("an error occurred during database migration")
+
 			return err
 		}
 	}
@@ -96,17 +126,19 @@ func (store *Store) connectionMigrateData(migratorParams *migrator.MigratorParam
 
 // backupVersion will backup the database or panic if any errors occur
 func (store *Store) backupVersion(migrator *migrator.Migrator) error {
-	migrateLog.Info("Backing up database prior to version upgrade...")
+	log.Info().Msg("backing up database prior to version upgrade")
 
 	options := getBackupRestoreOptions(store.commonBackupDir())
 
 	_, err := store.backupWithOptions(options)
 	if err != nil {
-		migrateLog.Error("An error occurred during database backup", err)
+		log.Error().Err(err).Msg("an error occurred during database backup")
+
 		removalErr := store.removeWithOptions(options)
 		if removalErr != nil {
-			migrateLog.Error("An error occurred during store removal prior to backup", err)
+			log.Error().Err(err).Msg("an error occurred during store removal prior to backup")
 		}
+
 		return err
 	}
 

api/datastore/migrate_data_test.go

@@ -1,13 +1,20 @@
 package datastore
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
-	"log"
+	"io"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/database/boltdb"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/rs/zerolog/log"
 )
 
 // testVersion is a helper which tests current store version against wanted version
@@ -22,8 +29,32 @@ func testVersion(store *Store, versionWant int, t *testing.T) {
 }
 
 func TestMigrateData(t *testing.T) {
+	snapshotTests := []struct {
+		testName string
+		srcPath  string
+		wantPath string
+	}{
+		{
+			testName: "migrate version 24 to latest",
+			srcPath:  "test_data/input_24.json",
+			wantPath: "test_data/output_24_to_latest.json",
+		},
+	}
+	for _, test := range snapshotTests {
+		t.Run(test.testName, func(t *testing.T) {
+			err := migrateDBTestHelper(t, test.srcPath, test.wantPath)
+			if err != nil {
+				t.Errorf(
+					"Failed migrating mock database %v: %v",
+					test.srcPath,
+					err,
+				)
+			}
+		})
+	}
+
 	t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) {
-		newStore, store, teardown := MustNewTestStore(false)
+		newStore, store, teardown := MustNewTestStore(t, false, true)
 		defer teardown()
 
 		if !newStore {
@@ -50,7 +81,7 @@ func TestMigrateData(t *testing.T) {
 		{version: 21, expectedVersion: portainer.DBVersion},
 	}
 	for _, tc := range tests {
-		_, store, teardown := MustNewTestStore(true)
+		_, store, teardown := MustNewTestStore(t, true, true)
 		defer teardown()
 
 		// Setup data
@@ -75,7 +106,7 @@ func TestMigrateData(t *testing.T) {
 	}
 
 	t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false)
+		_, store, teardown := MustNewTestStore(t, false, true)
 		defer teardown()
 
 		version := 17
@@ -87,7 +118,7 @@ func TestMigrateData(t *testing.T) {
 	})
 
 	t.Run("MigrateData should create backup file upon update", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false)
+		_, store, teardown := MustNewTestStore(t, false, true)
 		defer teardown()
 		store.VersionService.StoreDBVersion(0)
 
@@ -101,7 +132,7 @@ func TestMigrateData(t *testing.T) {
 	})
 
 	t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false)
+		_, store, teardown := MustNewTestStore(t, false, true)
 		defer teardown()
 
 		store.VersionService.StoreIsUpdating(true)
@@ -116,7 +147,7 @@ func TestMigrateData(t *testing.T) {
 	})
 
 	t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) {
-		_, store, teardown := MustNewTestStore(false)
+		_, store, teardown := MustNewTestStore(t, false, true)
 		defer teardown()
 
 		store.MigrateData()
@@ -127,48 +158,48 @@ func TestMigrateData(t *testing.T) {
 			t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath)
 		}
 	})
-
 }
 
 func Test_getBackupRestoreOptions(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false)
+	_, store, teardown := MustNewTestStore(t, false, true)
 	defer teardown()
 
 	options := getBackupRestoreOptions(store.commonBackupDir())
 
 	wantDir := store.commonBackupDir()
 	if !strings.HasSuffix(options.BackupDir, wantDir) {
-		log.Fatalf("incorrect backup dir; got=%s, want=%s", options.BackupDir, wantDir)
+		log.Fatal().Str("got", options.BackupDir).Str("want", wantDir).Msg("incorrect backup dir")
 	}
 
 	wantFilename := "portainer.db.bak"
 	if options.BackupFileName != wantFilename {
-		log.Fatalf("incorrect backup file; got=%s, want=%s", options.BackupFileName, wantFilename)
+		log.Fatal().Str("got", options.BackupFileName).Str("want", wantFilename).Msg("incorrect backup file")
 	}
 }
 
 func TestRollback(t *testing.T) {
 	t.Run("Rollback should restore upgrade after backup", func(t *testing.T) {
 		version := 21
-		_, store, teardown := MustNewTestStore(false)
+		_, store, teardown := MustNewTestStore(t, false, true)
 		defer teardown()
 		store.VersionService.StoreDBVersion(version)
 
 		_, err := store.backupWithOptions(getBackupRestoreOptions(store.commonBackupDir()))
 		if err != nil {
-			log.Fatal(err)
+			log.Fatal().Err(err).Msg("")
 		}
 
 		// Change the current edition
 		err = store.VersionService.StoreDBVersion(version + 10)
 		if err != nil {
-			log.Fatal(err)
+			log.Fatal().Err(err).Msg("")
 		}
 
 		err = store.Rollback(true)
 		if err != nil {
 			t.Logf("Rollback failed: %s", err)
 			t.Fail()
+
 			return
 		}
 
@@ -185,3 +216,250 @@ func isFileExist(path string) bool {
 	}
 	return len(matches) > 0
 }
+
+// migrateDBTestHelper loads a json representation of a bolt database from srcPath,
+// parses it into a database, runs a migration on that database, and then
+// compares it with an expected output database.
+func migrateDBTestHelper(t *testing.T, srcPath, wantPath string) error {
+	srcJSON, err := os.ReadFile(srcPath)
+	if err != nil {
+		t.Fatalf("failed loading source JSON file %v: %v", srcPath, err)
+	}
+
+	// Parse source json to db.
+	_, store, teardown := MustNewTestStore(t, true, false)
+	defer teardown()
+	err = importJSON(t, bytes.NewReader(srcJSON), store)
+	if err != nil {
+		return err
+	}
+
+	// Run the actual migrations on our input database.
+	err = store.MigrateData()
+	if err != nil {
+		return err
+	}
+
+	// Assert that our database connection is using bolt so we can call
+	// exportJson rather than ExportRaw. The exportJson function allows us to
+	// strip out the metadata which we don't want for our tests.
+	// TODO: update connection interface in CE to allow us to use ExportRaw and pass meta false
+	err = store.connection.Close()
+	if err != nil {
+		t.Fatalf("err closing bolt connection: %v", err)
+	}
+	con, ok := store.connection.(*boltdb.DbConnection)
+	if !ok {
+		t.Fatalf("backing database is not using boltdb, but the migrations test requires it")
+	}
+
+	// Convert database back to json.
+	databasePath := con.GetDatabaseFilePath()
+	if _, err := os.Stat(databasePath); err != nil {
+		return fmt.Errorf("stat on %s failed: %s", databasePath, err)
+	}
+
+	gotJSON, err := con.ExportJson(databasePath, false)
+	if err != nil {
+		t.Logf(
+			"failed re-exporting database %s to JSON: %v",
+			databasePath,
+			err,
+		)
+	}
+
+	wantJSON, err := os.ReadFile(wantPath)
+	if err != nil {
+		t.Fatalf("failed loading want JSON file %v: %v", wantPath, err)
+	}
+
+	// Compare the result we got with the one we wanted.
+	if diff := cmp.Diff(wantJSON, gotJSON); diff != "" {
+		gotPath := filepath.Join(os.TempDir(), "portainer-migrator-test-fail.json")
+		os.WriteFile(
+			gotPath,
+			gotJSON,
+			0600,
+		)
+		t.Errorf(
+			"migrate data from %s to %s failed\nwrote migrated input to %s\nmismatch (-want +got):\n%s",
+			srcPath,
+			wantPath,
+			gotPath,
+			diff,
+		)
+	}
+	return nil
+}
+
+// importJSON reads input JSON and commits it to a portainer datastore.Store.
+// Errors are logged with the testing package.
+func importJSON(t *testing.T, r io.Reader, store *Store) error {
+	objects := make(map[string]interface{})
+
+	// Parse json into map of objects.
+	d := json.NewDecoder(r)
+	d.UseNumber()
+	err := d.Decode(&objects)
+	if err != nil {
+		return err
+	}
+
+	// Get database connection from store.
+	con := store.connection
+
+	for k, v := range objects {
+		switch k {
+		case "version":
+			versions, ok := v.(map[string]interface{})
+			if !ok {
+				t.Logf("failed casting %s to map[string]interface{}", k)
+			}
+
+			dbVersion, ok := versions["DB_VERSION"]
+			if !ok {
+				t.Logf("failed getting DB_VERSION from %s", k)
+			}
+
+			numDBVersion, ok := dbVersion.(json.Number)
+			if !ok {
+				t.Logf("failed parsing DB_VERSION as json number from %s", k)
+			}
+
+			intDBVersion, err := numDBVersion.Int64()
+			if err != nil {
+				t.Logf("failed casting %v to int: %v", numDBVersion, intDBVersion)
+			}
+
+			err = con.CreateObjectWithStringId(
+				k,
+				[]byte("DB_VERSION"),
+				int(intDBVersion),
+			)
+			if err != nil {
+				t.Logf("failed writing DB_VERSION in %s: %v", k, err)
+			}
+
+			instanceID, ok := versions["INSTANCE_ID"]
+			if !ok {
+				t.Logf("failed getting INSTANCE_ID from %s", k)
+			}
+
+			err = con.CreateObjectWithStringId(
+				k,
+				[]byte("INSTANCE_ID"),
+				instanceID,
+			)
+			if err != nil {
+				t.Logf("failed writing INSTANCE_ID in %s: %v", k, err)
+			}
+
+		case "dockerhub":
+			obj, ok := v.([]interface{})
+			if !ok {
+				t.Logf("failed to cast %s to []interface{}", k)
+			}
+			err := con.CreateObjectWithStringId(
+				k,
+				[]byte("DOCKERHUB"),
+				obj[0],
+			)
+			if err != nil {
+				t.Logf("failed writing DOCKERHUB in %s: %v", k, err)
+			}
+
+		case "ssl":
+			obj, ok := v.(map[string]interface{})
+			if !ok {
+				t.Logf("failed to case %s to map[string]interface{}", k)
+			}
+			err := con.CreateObjectWithStringId(
+				k,
+				[]byte("SSL"),
+				obj,
+			)
+			if err != nil {
+				t.Logf("failed writing SSL in %s: %v", k, err)
+			}
+
+		case "settings":
+			obj, ok := v.(map[string]interface{})
+			if !ok {
+				t.Logf("failed to case %s to map[string]interface{}", k)
+			}
+			err := con.CreateObjectWithStringId(
+				k,
+				[]byte("SETTINGS"),
+				obj,
+			)
+			if err != nil {
+				t.Logf("failed writing SETTINGS in %s: %v", k, err)
+			}
+
+		case "tunnel_server":
+			obj, ok := v.(map[string]interface{})
+			if !ok {
+				t.Logf("failed to case %s to map[string]interface{}", k)
+			}
+			err := con.CreateObjectWithStringId(
+				k,
+				[]byte("INFO"),
+				obj,
+			)
+			if err != nil {
+				t.Logf("failed writing INFO in %s: %v", k, err)
+			}
+		case "templates":
+			continue
+
+		default:
+			objlist, ok := v.([]interface{})
+			if !ok {
+				t.Logf("failed to cast %s to []interface{}", k)
+			}
+
+			for _, obj := range objlist {
+				value, ok := obj.(map[string]interface{})
+				if !ok {
+					t.Logf("failed to cast %v to map[string]interface{}", obj)
+				} else {
+					var ok bool
+					var id interface{}
+					switch k {
+					case "endpoint_relations":
+						// TODO: need to make into an int, then do that weird
+						// stringification
+						id, ok = value["EndpointID"]
+					default:
+						id, ok = value["Id"]
+					}
+					if !ok {
+						// endpoint_relations: EndpointID
+						t.Logf("missing Id field: %s", k)
+						id = "error"
+					}
+					n, ok := id.(json.Number)
+					if !ok {
+						t.Logf("failed to cast %v to json.Number in %s", id, k)
+					} else {
+						key, err := n.Int64()
+						if err != nil {
+							t.Logf("failed to cast %v to int in %s", n, k)
+						} else {
+							err := con.CreateObjectWithId(
+								k,
+								int(key),
+								value,
+							)
+							if err != nil {
+								t.Logf("failed writing %v in %s: %v", key, k, err)
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}

api/datastore/migrate_dbversion29_test.go

@@ -33,7 +33,7 @@ func setup(store *Store) error {
 }
 
 func TestMigrateSettings(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false)
+	_, store, teardown := MustNewTestStore(t, false, true)
 	defer teardown()
 
 	err := setup(store)

api/datastore/migrate_dbversion33_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestMigrateStackEntryPoint(t *testing.T) {
-	_, store, teardown := MustNewTestStore(false)
+	_, store, teardown := MustNewTestStore(t, false, true)
 	defer teardown()
 
 	stackService := store.Stack()

api/datastore/migrator/migrate_ce.go

@@ -1,16 +1,40 @@
 package migrator
 
 import (
-	"fmt"
+	"errors"
+	"reflect"
+	"runtime"
+
+	portainer "github.com/portainer/portainer/api"
 
 	werrors "github.com/pkg/errors"
-	portainer "github.com/portainer/portainer/api"
+	"github.com/rs/zerolog/log"
 )
 
+type migration struct {
+	dbversion int
+	migrate   func() error
+}
+
 func migrationError(err error, context string) error {
 	return werrors.Wrap(err, "failed in "+context)
 }
 
+func newMigration(dbversion int, migrate func() error) migration {
+	return migration{
+		dbversion: dbversion,
+		migrate:   migrate,
+	}
+}
+
+func dbTooOldError() error {
+	return errors.New("migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support.")
+}
+
+func GetFunctionName(i interface{}) string {
+	return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
+}
+
 // Migrate checks the database version and migrate the existing data to the most recent data model.
 func (m *Migrator) Migrate() error {
 	// set DB to updating status
@@ -19,175 +43,104 @@ func (m *Migrator) Migrate() error {
 		return migrationError(err, "StoreIsUpdating")
 	}
 
-	if m.currentDBVersion < 17 {
-		return migrationError(err, "migrating from less than Portainer 1.21.0 is not supported, please contact Portainer support.")
+	migrations := []migration{
+		// Portainer < 1.21.0
+		newMigration(17, dbTooOldError),
+
+		// Portainer 1.21.0
+		newMigration(18, m.updateUsersToDBVersion18),
+		newMigration(18, m.updateEndpointsToDBVersion18),
+		newMigration(18, m.updateEndpointGroupsToDBVersion18),
+		newMigration(18, m.updateRegistriesToDBVersion18),
+
+		// 1.22.0
+		newMigration(19, m.updateSettingsToDBVersion19),
+
+		// 1.22.1
+		newMigration(20, m.updateUsersToDBVersion20),
+		newMigration(20, m.updateSettingsToDBVersion20),
+		newMigration(20, m.updateSchedulesToDBVersion20),
+
+		// Portainer 1.23.0
+		// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
+		newMigration(22, m.updateResourceControlsToDBVersion22),
+		newMigration(22, m.updateUsersAndRolesToDBVersion22),
+
+		// Portainer 1.24.0
+		newMigration(23, m.updateTagsToDBVersion23),
+		newMigration(23, m.updateEndpointsAndEndpointGroupsToDBVersion23),
+
+		// Portainer 1.24.1
+		newMigration(24, m.updateSettingsToDB24),
+
+		// Portainer 2.0.0
+		newMigration(25, m.updateSettingsToDB25),
+		newMigration(25, m.updateStacksToDB24), // yes this looks odd. Don't be tempted to move it
+
+		// Portainer 2.1.0
+		newMigration(26, m.updateEndpointSettingsToDB25),
+
+		// Portainer 2.2.0
+		newMigration(27, m.updateStackResourceControlToDB27),
+
+		// Portainer 2.6.0
+		newMigration(30, m.migrateDBVersionToDB30),
+
+		// Portainer 2.9.0
+		newMigration(32, m.migrateDBVersionToDB32),
+
+		// Portainer 2.9.1, 2.9.2
+		newMigration(33, m.migrateDBVersionToDB33),
+
+		// Portainer 2.10
+		newMigration(34, m.migrateDBVersionToDB34),
+
+		// Portainer 2.9.3 (yep out of order, but 2.10 is EE only)
+		newMigration(35, m.migrateDBVersionToDB35),
+
+		newMigration(36, m.migrateDBVersionToDB36),
+
+		// Portainer 2.13
+		newMigration(40, m.migrateDBVersionToDB40),
+
+		// Portainer 2.14
+		newMigration(50, m.migrateDBVersionToDB50),
+
+		// Portainer 2.15
+		newMigration(60, m.migrateDBVersionToDB60),
+
+		// Portainer 2.16
+		newMigration(70, m.migrateDBVersionToDB70),
+
+		// Portainer 2.16.1
+		newMigration(71, m.migrateDBVersionToDB71),
 	}
 
-	// Portainer 1.21.0
-	if m.currentDBVersion < 18 {
-		err := m.updateUsersToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateUsersToDBVersion18")
-		}
+	var lastDbVersion int
+	for _, migration := range migrations {
+		if m.currentDBVersion < migration.dbversion {
 
-		err = m.updateEndpointsToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateEndpointsToDBVersion18")
-		}
+			// Print the next line only when the version changes
+			if migration.dbversion > lastDbVersion {
+				log.Info().Int("to_version", migration.dbversion).Msg("migrating DB")
+			}
 
-		err = m.updateEndpointGroupsToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateEndpointGroupsToDBVersion18")
-		}
-
-		err = m.updateRegistriesToDBVersion18()
-		if err != nil {
-			return migrationError(err, "updateRegistriesToDBVersion18")
+			err := migration.migrate()
+			if err != nil {
+				return migrationError(err, GetFunctionName(migration.migrate))
+			}
 		}
+		lastDbVersion = migration.dbversion
 	}
 
-	// Portainer 1.22.0
-	if m.currentDBVersion < 19 {
-		err := m.updateSettingsToDBVersion19()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion19")
-		}
-	}
-
-	// Portainer 1.22.1
-	if m.currentDBVersion < 20 {
-		err := m.updateUsersToDBVersion20()
-		if err != nil {
-			return migrationError(err, "updateUsersToDBVersion20")
-		}
-
-		err = m.updateSettingsToDBVersion20()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDBVersion20")
-		}
-
-		err = m.updateSchedulesToDBVersion20()
-		if err != nil {
-			return migrationError(err, "updateSchedulesToDBVersion20")
-		}
-	}
-
-	// Portainer 1.23.0
-	// DBVersion 21 is missing as it was shipped as via hotfix 1.22.2
-	if m.currentDBVersion < 22 {
-		err := m.updateResourceControlsToDBVersion22()
-		if err != nil {
-			return migrationError(err, "updateResourceControlsToDBVersion22")
-		}
-
-		err = m.updateUsersAndRolesToDBVersion22()
-		if err != nil {
-			return migrationError(err, "updateUsersAndRolesToDBVersion22")
-		}
-	}
-
-	// Portainer 1.24.0
-	if m.currentDBVersion < 23 {
-		migrateLog.Info("Migrating to DB 23")
-		err := m.updateTagsToDBVersion23()
-		if err != nil {
-			return migrationError(err, "updateTagsToDBVersion23")
-		}
-
-		err = m.updateEndpointsAndEndpointGroupsToDBVersion23()
-		if err != nil {
-			return migrationError(err, "updateEndpointsAndEndpointGroupsToDBVersion23")
-		}
-	}
-
-	// Portainer 1.24.1
-	if m.currentDBVersion < 24 {
-		migrateLog.Info("Migrating to DB 24")
-		err := m.updateSettingsToDB24()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDB24")
-		}
-	}
-
-	// Portainer 2.0.0
-	if m.currentDBVersion < 25 {
-		migrateLog.Info("Migrating to DB 25")
-		err := m.updateSettingsToDB25()
-		if err != nil {
-			return migrationError(err, "updateSettingsToDB25")
-		}
-
-		err = m.updateStacksToDB24()
-		if err != nil {
-			return migrationError(err, "updateStacksToDB24")
-		}
-	}
-
-	// Portainer 2.1.0
-	if m.currentDBVersion < 26 {
-		migrateLog.Info("Migrating to DB 26")
-		err := m.updateEndpointSettingsToDB25()
-		if err != nil {
-			return migrationError(err, "updateEndpointSettingsToDB25")
-		}
-	}
-
-	// Portainer 2.2.0
-	if m.currentDBVersion < 27 {
-		migrateLog.Info("Migrating to DB 27")
-		err := m.updateStackResourceControlToDB27()
-		if err != nil {
-			return migrationError(err, "updateStackResourceControlToDB27")
-		}
-	}
-
-	// Portainer 2.6.0
-	if m.currentDBVersion < 30 {
-		migrateLog.Info("Migrating to DB 30")
-		err := m.migrateDBVersionToDB30()
-		if err != nil {
-			return migrationError(err, "migrateDBVersionToDB30")
-		}
-	}
-
-	// Portainer 2.9.0
-	if m.currentDBVersion < 32 {
-		err := m.migrateDBVersionToDB32()
-		if err != nil {
-			return migrationError(err, "migrateDBVersionToDB32")
-		}
-	}
-
-	// Portainer 2.9.1, 2.9.2
-	if m.currentDBVersion < 33 {
-		migrateLog.Info("Migrating to DB 33")
-		err := m.migrateDBVersionToDB33()
-		if err != nil {
-			return migrationError(err, "migrateDBVersionToDB33")
-		}
-	}
-
-	// Portainer 2.10
-	if m.currentDBVersion < 34 {
-		migrateLog.Info("Migrating to DB 34")
-		if err := m.migrateDBVersionToDB34(); err != nil {
-			return migrationError(err, "migrateDBVersionToDB34")
-		}
-	}
-
-	// Portainer 2.9.3 (yep out of order, but 2.10 is EE only)
-	if m.currentDBVersion < 35 {
-		migrateLog.Info("Migrating to DB 35")
-		if err := m.migrateDBVersionToDB35(); err != nil {
-			return migrationError(err, "migrateDBVersionToDB35")
-		}
-	}
+	log.Info().Int("version", portainer.DBVersion).Msg("setting DB version")
 
 	err = m.versionService.StoreDBVersion(portainer.DBVersion)
 	if err != nil {
 		return migrationError(err, "StoreDBVersion")
 	}
-	migrateLog.Info(fmt.Sprintf("Updated DB version to %d", portainer.DBVersion))
+
+	log.Info().Int("version", portainer.DBVersion).Msg("updated DB version")
 
 	// reset DB updating status
 	return m.versionService.StoreIsUpdating(false)

api/datastore/migrator/migrate_dbversion17.go

@@ -2,9 +2,13 @@ package migrator
 
 import (
 	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) updateUsersToDBVersion18() error {
+	log.Info().Msg("updating users")
+
 	legacyUsers, err := m.userService.Users()
 	if err != nil {
 		return err
@@ -39,6 +43,8 @@ func (m *Migrator) updateUsersToDBVersion18() error {
 }
 
 func (m *Migrator) updateEndpointsToDBVersion18() error {
+	log.Info().Msg("updating endpoints")
+
 	legacyEndpoints, err := m.endpointService.Endpoints()
 	if err != nil {
 		return err
@@ -69,6 +75,8 @@ func (m *Migrator) updateEndpointsToDBVersion18() error {
 }
 
 func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
+	log.Info().Msg("updating endpoint groups")
+
 	legacyEndpointGroups, err := m.endpointGroupService.EndpointGroups()
 	if err != nil {
 		return err
@@ -99,6 +107,8 @@ func (m *Migrator) updateEndpointGroupsToDBVersion18() error {
 }
 
 func (m *Migrator) updateRegistriesToDBVersion18() error {
+	log.Info().Msg("updating registries")
+
 	legacyRegistries, err := m.registryService.Registries()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion18.go

@@ -1,8 +1,14 @@
 package migrator
 
-import portainer "github.com/portainer/portainer/api"
+import (
+	portainer "github.com/portainer/portainer/api"
+
+	"github.com/rs/zerolog/log"
+)
 
 func (m *Migrator) updateSettingsToDBVersion19() error {
+	log.Info().Msg("updating settings")
+
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion19.go

@@ -2,11 +2,15 @@ package migrator
 
 import (
 	"strings"
+
+	"github.com/rs/zerolog/log"
 )
 
 const scheduleScriptExecutionJobType = 1
 
 func (m *Migrator) updateUsersToDBVersion20() error {
+	log.Info().Msg("updating user authentication")
+
 	return m.authorizationService.UpdateUsersAuthorizations()
 }
 
@@ -22,6 +26,8 @@ func (m *Migrator) updateSettingsToDBVersion20() error {
 }
 
 func (m *Migrator) updateSchedulesToDBVersion20() error {
+	log.Info().Msg("updating schedules")
+
 	legacySchedules, err := m.scheduleService.Schedules()
 	if err != nil {
 		return err

api/datastore/migrator/migrate_dbversion20.go

@@ -3,9 +3,13 @@ package migrator
 import (
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/internal/authorization"
+
+	"github.com/rs/zerolog/log"
 )
 
 func (m *Migrator) updateResourceControlsToDBVersion22() error {
+	log.Info().Msg("updating resource controls")
+
 	legacyResourceControls, err := m.resourceControlService.ResourceControls()
 	if err != nil {
 		return err
@@ -24,6 +28,8 @@ func (m *Migrator) updateResourceControlsToDBVersion22() error {
 }
 
 func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
+	log.Info().Msg("updating users and roles")
+
 	legacyUsers, err := m.userService.Users()
 	if err != nil {
 		return err