Compare commits

...

112 Commits

Author SHA1 Message Date
deviantony
ababd63d97 feat: update semver matching 2024-10-04 07:57:31 +00:00
James Carppe
b40d22dc74 Update bug report template for 2.22.0 (#12283) 2024-10-03 14:53:37 +13:00
Steven Kang
a257696c25 fix access conditions when the restrict default namespace is enabled (#12280) 2024-10-02 15:55:05 +13:00
andres-portainer
f742937359 fix(endpoints): optimize the search performance BE-11267 (#12262) 2024-10-01 15:13:54 -03:00
Steven Kang
c0db48b29d fix ingress creation for none class (#12273) 2024-10-01 14:43:46 +13:00
Steven Kang
ea228c3d6d refactor(k8s): namespace core logic (#12142)
Co-authored-by: testA113 <aliharriss1995@gmail.com>
Co-authored-by: Anthony Lapenna <anthony.lapenna@portainer.io>
Co-authored-by: James Carppe <85850129+jamescarppe@users.noreply.github.com>
Co-authored-by: Ali <83188384+testA113@users.noreply.github.com>
2024-10-01 14:15:51 +13:00
Ali
da010f3d08 fix(podman): ensure initial env type matches container runtime [r8s-98] (#12259) 2024-09-30 09:16:24 +13:00
Ali
32e94d4e4e feat(podman): support add podman envs in the wizard [r8s-20] (#12056) 2024-09-25 11:55:07 +12:00
Ali
db616bc8a5 fix(wizard): update nodeport placeholder [r8s-62] (#12255) 2024-09-25 11:36:50 +12:00
James Carppe
b8b46ec129 Update bug report template for 2.21.2 (#12251) 2024-09-24 11:42:15 +12:00
LP B
7d0b79a546 fix(app/images): export images to tar (#12223) 2024-09-23 21:55:45 +02:00
LP B
fd26565b14 fix(app/templates): non admins cannot load templates list (#12235) 2024-09-23 17:54:32 +02:00
Nik Wakelin
e0b6f2283a chore(branding): Changes Linode to Akamai Connected Cloud (#12221) 2024-09-23 09:21:02 +12:00
Oscar Zhou
d3d3d50569 fix(version): add specific version for updater image [BE-11153] (#12227) 2024-09-21 14:54:08 +12:00
andres-portainer
cee997e0b3 fix(edgestacks): reorder operations to properly update the endpoint relations BE-11233 (#12239) 2024-09-20 19:10:28 -03:00
LP B
80f53ed6ec fix(api): skip guessing env when there is no env in DB (#12238) 2024-09-20 17:56:41 -03:00
Chaim Lev-Ari
6f84317e7a feat(system): upgrade on swarm [EE-5848] (#11728)
Co-authored-by: Chaim Lev-Ari <chaim.levi-ari@portainer.io>
Co-authored-by: LP B <xAt0mZ@users.noreply.github.com>
2024-09-20 18:00:38 +02:00
LP B
3cb484f06a fix(app/users): password validation hint + missing message on empty teams list (#12231) 2024-09-20 16:33:13 +02:00
LP B
61353cbe8a fix(app/edge): race between redirects when selecting a template (#12230) 2024-09-20 16:00:40 +02:00
Yajith Dayarathna
d647980c3a updating attest params (#12228) 2024-09-20 11:48:32 +12:00
Oscar Zhou
5740abe31b fix(authorization): add registry button disappear for admin [BE-11228] (#12213) 2024-09-20 08:18:51 +12:00
andres-portainer
5fd4f52e35 fix(jwt): fix handling of non-expiring JWT tokens BE-11242 (#12220) 2024-09-17 18:23:33 -03:00
Yajith Dayarathna
dbe7cd16d4 2024-09-CVE (#12189) 2024-09-11 11:08:46 +12:00
Yajith Dayarathna
2b630ca2dd enabling build attestations (#12211) 2024-09-11 10:57:52 +12:00
Oscar Zhou
2ede22646b fix(version): add specific version for updater image [BE-11153] (#12202) 2024-09-11 08:29:23 +12:00
James Carppe
994b6bb471 Update bug report template for 2.21.1 (#12207) 2024-09-10 14:33:32 +12:00
andres-portainer
92f338e0cd fix(users): fix data-race in userCreate() BE-11209 (#12193) 2024-09-05 22:28:04 -03:00
andres-portainer
7a176cf284 fix(teams): fix data-race in teamCreate() BE-11210 (#12195) 2024-09-05 21:36:13 -03:00
Oscar Zhou
80e607ab30 fix(stack): env placeholder as host path [BE-11187] (#12192) 2024-09-06 08:43:12 +12:00
Anthony Lapenna
6cff21477e service: update stop grace period description (#12173) 2024-09-05 08:47:06 +02:00
Yajith Dayarathna
4bb5a7f480 updating ci workflow (#12183) 2024-09-05 09:19:36 +12:00
andres-portainer
9a88511d00 fix(docker): avoid specifying the MAC address of container for Docker API < v1.44 BE-10880 (#12179) 2024-09-03 10:31:24 -03:00
Yajith Dayarathna
48cd614948 CVE 2024 43798 (#12171) 2024-09-03 09:27:24 +12:00
andres-portainer
2fe252d62b fix(jwt): generate JWT IDs BE-11179 (#12175) 2024-09-02 12:06:39 -03:00
LP B
8fae7f8438 feat(app/wizard): info panel telling to add env only once per swarm cluster (#11954) 2024-09-02 14:22:07 +02:00
andres-portainer
e4e55157e8 fix(bouncer): add support for JWT revocation BE-11179 (#12164) 2024-08-30 20:24:05 -03:00
Yajith Dayarathna
a5e246cc16 testing go directive change (#12124) 2024-08-30 08:27:42 +02:00
andres-portainer
d28dc59584 fix(git): optimize listFiles() BE-11184 (#12160) 2024-08-29 19:01:51 -03:00
andres-portainer
5353570721 task(code): remove unnecessary uses of govalidator BE-11181 (#12156) 2024-08-28 19:37:20 -03:00
andres-portainer
eb3e367ba8 fix(edgestacks): change the level of a logged line EE-6874 (#11396) 2024-08-28 18:16:34 -03:00
Chaim Lev-Ari
3c1441d462 refactor(users): migrate list view to react [EE-2202] (#11914) 2024-08-28 17:04:32 -03:00
Chaim Lev-Ari
33ce841040 refactor(docker/events): migrate list view to react [EE-2228] (#11581) 2024-08-28 16:41:15 -03:00
Chaim Lev-Ari
9797201c2a feat(docker): label gpu as nvidia only [EE-6999] (#11729) 2024-08-28 16:38:27 -03:00
Chaim Lev-Ari
6e14ac583b fix(access-control): fix dt column header typo [EE-7113] (#11853) 2024-08-28 16:37:12 -03:00
Anthony Lapenna
0b37b677c1 refactor: fix linting issues across the codebase (#12152) 2024-08-28 15:03:15 +02:00
Oscar Zhou
f59dd34154 fix(swarm/service): list task when filtering service [BE-11029] (#12146) 2024-08-28 18:28:38 +12:00
James Carppe
e8ec648886 Update bug report template for 2.21.0 (#12145) 2024-08-27 16:42:49 +12:00
Ali
10767a06df fix(invalidate): keep invalidate default behaviour [BE-11064] (#12080) 2024-08-27 09:48:50 +12:00
James Carppe
59b3375b59 Update bug report template for 2.21.0-rc2 (#12128) 2024-08-23 10:55:43 +12:00
andres-portainer
4408fd0cd3 chore(polling): simplify the polling logic BE-4585 (#12121) 2024-08-22 10:54:34 -03:00
Yajith Dayarathna
975a9517b9 undo change to go directive 2024-08-22 16:21:13 +12:00
Yajith Dayarathna
89c92b7834 updating go directive 2024-08-22 16:17:28 +12:00
Anthony Lapenna
747cea8084 security: bump dependencies to address CVEs (#12119) 2024-08-21 20:08:25 +12:00
Ali
f016b31388 fix(docker-desktop): support auth cookies [BE-11134] (#12108) 2024-08-21 18:21:51 +12:00
Oscar Zhou
8cd53a4b7a fix(registry): non admin can see add registry button [BE-10834] (#12112) 2024-08-21 11:00:00 +12:00
LP B
a39abe61c2 fix(api/edge_stacks): ensure edge stacks related endpoints list generation returns unique elements (#12101) 2024-08-20 10:20:03 +02:00
James Carppe
054898f821 Update bug report template for 2.21.0-rc1 (#12104) 2024-08-15 19:27:24 +12:00
Oscar Zhou
13d9b12a2e fix(group): create group twice when associating devices [EE-7418] (#12092) 2024-08-12 17:09:49 +12:00
LP B
aaec856282 fix(app/registries): enforce user accesses on registries (#12087) 2024-08-10 11:53:16 +02:00
andres-portainer
009eec9475 fix(compose): avoid the need to pass the file to remove the stack BE-11057 (#12065)
Co-authored-by: andres-portainer <andres-portainer@users.noreply.github.com>
Co-authored-by: Yajith Dayarathna <yajith.dayarathna@portainer.io>
2024-08-09 10:22:31 -03:00
Yajith Dayarathna
8d14535fd5 updating github workflow 2024-08-09 14:58:20 +12:00
Oscar Zhou
cc7f14951c fix(stack/remote): pass forceRecreate setting [EE-7374] (#12051) 2024-08-06 09:02:21 +12:00
Yajith Dayarathna
b67ff87f35 Installing docker-compose during test-server step (#12075) 2024-08-05 11:28:47 +12:00
andres-portainer
f55ef6e691 fix(pendingactions): remove excessive logging BE-11094 (#12071) 2024-08-02 16:35:14 -03:00
andres-portainer
560a1a00ca fix(scheduler): remove jobs that won't be used anymore BE-11045 (#12058) 2024-08-01 10:59:29 -03:00
andres-portainer
3b5ce1b053 fix(scheduler): remove unnecessary goroutines BE-11044 (#12059) 2024-08-01 10:58:53 -03:00
andres-portainer
03e8d05f18 fix(scheduler): fix a data race in a unit test BE-11084 (#12057) 2024-08-01 10:58:08 -03:00
Oscar Zhou
bedb7fb255 fix(swarm): auto multi-select volume with same name [EE-7240] (#11955) 2024-07-31 12:12:26 +12:00
Oscar Zhou
4d586f7a85 fix(docker): missing browse volume option [EE-7179] (#11901) 2024-07-30 08:53:17 +12:00
Oscar Zhou
6486a5d971 fix(edgestack): broken parallel setting in create view [EE-7245] (#11945) 2024-07-29 09:42:05 +12:00
andres-portainer
e3364457c4 fix(security): update github.com/containers/image/v5 to fix GO-2024-2842 BE-11055 (#12046) 2024-07-23 18:56:17 -03:00
andres-portainer
66119a8b57 fix(snapshots): remove the attempt to snapshot untrusted environments EE-7407 (#12044) 2024-07-23 18:43:31 -03:00
Ali
6eb9e906af fix(placements) filter out empty items in the required node affinity array [BE-11022] (#12034)
Co-authored-by: testa113 <testa113>
2024-07-23 09:31:08 +12:00
LP B
1900fb695d fix(docker/container): use nodeName to build links to networks used by containers (#12002) 2024-07-17 14:40:05 +02:00
Oscar Zhou
a62aac296b fix(host): show clear host info message [EE-7075] (#12011) 2024-07-12 08:45:50 +12:00
Oscar Zhou
5294aa2810 fix(stack): excessive alias count error [EE-7305] (#11990) 2024-07-11 14:09:25 +12:00
andres-portainer
31bdb948a8 chore(code): use int ranges in loops BE-10990 (#12028) 2024-07-10 19:22:47 -03:00
andres-portainer
468c12c75b chore(bbolt): update to v1.3.10 EE-7298 (#12006) 2024-07-09 15:19:27 -03:00
andres-portainer
220fe28830 fix(snapshots): fix background snapshots on environment creation EE-7273 (#12021) 2024-07-09 15:18:13 -03:00
andres-portainer
7fd1a644a6 chore(loops): remove loop var copy EE-7342 (#12023) 2024-07-09 12:47:34 -03:00
andres-portainer
6e7a42727a chore(kompose): remove the code EE-4917 (#12003) 2024-07-08 17:19:07 -03:00
LP B
ac4b129195 fix(docker/network): send target nodeName when removing a network on swarm (#12001) 2024-07-08 17:31:18 +02:00
Steven Kang
85bc14e470 fix(cve): remediate cves detected in docker scout (#12018) 2024-07-08 10:24:39 +12:00
Yajith Dayarathna
6e791a2cfe (fix)nightly code security scan (#12017) 2024-07-06 10:54:41 +12:00
andres-portainer
340830d121 chore(docker): clean up the code EE-7325 (#11997) 2024-07-05 16:19:46 -03:00
andres-portainer
faca64442f chore(code): use cmp.Or() EE-7333 (#12009) 2024-07-04 19:23:53 -03:00
andres-portainer
854474478c chore(go): update to Go v1.22.5 EE-7297 (#12005) 2024-07-03 22:29:20 -03:00
andres-portainer
4adce14485 fix(errors): improve error handling EE-4430 (#11987) 2024-06-28 17:35:26 -03:00
andres-portainer
dc62604ed8 chore(code): remove unused third-party code EE-7306 (#11988) 2024-06-28 17:33:53 -03:00
andres-portainer
f0d43f941f chore(code): replace interface{} with any EE-6513 (#11986) 2024-06-28 14:59:28 -03:00
andres-portainer
9c4935286f chore(jsoniter): remove leftover code EE-6702 (#11984) 2024-06-28 09:46:49 -03:00
andres-portainer
e1648425ea chore(nomad): remove Nomad code EE-7234 (#11980) 2024-06-28 08:54:44 -03:00
andres-portainer
19fa40286a chore(fdo): remove FDO code EE-7235 (#11981) 2024-06-28 08:42:16 -03:00
andres-portainer
1a3db327c7 fix(kube): improve error handling EE-7196 (#11976) 2024-06-27 10:45:11 -03:00
andres-portainer
1170004097 fix(kube): improve error handling EE-7199 (#11974) 2024-06-27 10:43:44 -03:00
Ali
d2b0eacbf5 chore(deps): upgrade typescript to 5.5 [EE-7294] (#11970)
Co-authored-by: testa113 <testa113>
2024-06-27 13:54:10 +12:00
andres-portainer
ca9f85a1ff fix(snapshots): enable the background snapshotter EE-7273 (#11973) 2024-06-26 18:27:39 -03:00
andres-portainer
9ee092aa5e chore(code): reduce the code duplication EE-7278 (#11969) 2024-06-26 18:14:22 -03:00
Oscar Zhou
39bdfa4512 fix(edgestack): gitops auto update issue [EE-7260] (#11965) 2024-06-27 08:48:03 +12:00
cmeng
e828615467 fix(host-info) host info improvement EE-7075 (#11884) 2024-06-26 12:18:22 -03:00
Ali
ba4526985a fix(app): show services explanation in app form [EE-7284] (#11967)
Co-authored-by: testa113 <testa113>
2024-06-26 10:44:20 +12:00
Chaim Lev-Ari
607feb183e refactor(ui/button): remove duplicate data-cy [EE-7242] (#11934) 2024-06-20 15:39:03 +02:00
LP B
9994ed157a fix(app): properly update the app state when losing connectivity to a remote environment while browsing it (#11942) 2024-06-19 13:44:57 +02:00
andres-portainer
bfa27d9103 chore(code): clean up the code EE-7251 (#11948) 2024-06-18 15:59:12 -03:00
Ali
be9d3285e1 fix(custom-templates): add stack validation, remove custom template validation [EE-7102] (#11938)
Co-authored-by: testa113 <testa113>
2024-06-17 09:24:54 +12:00
Chaim Lev-Ari
0f5988af49 fix(edge/stacks): load template [EE-7109] (#11848) 2024-06-16 07:54:00 +03:00
Chaim Lev-Ari
a28bd349ae fix(edge/update): show environment count when more than 100 [EE-6424] (#11917) 2024-06-14 18:37:45 -03:00
Chaim Lev-Ari
51f9977885 fix(endpoints): show toaster on delete [EE-7170] (#11889) 2024-06-13 18:32:17 -03:00
Ali
27865981df fix(namespace): sanitize owner label [EE-7122] (#11935)
Co-authored-by: testa113 <testa113>
2024-06-13 11:06:17 +12:00
James Carppe
ac3f1cd5c3 Add support for specifying the NFS server address in the mount point EE-7019 (#11921) 2024-06-12 11:23:08 -03:00
Dakota Walsh
7549b6cf3f fix(kubernetes): cluster setup screen text on own line EE-7112 (#11905) 2024-06-12 08:43:17 +12:00
Oscar Zhou
dd372ee122 fix(customtemplate): duplicated error handling [EE-7197] (#11913) 2024-06-11 22:11:15 +12:00
875 changed files with 17328 additions and 11263 deletions

View File

@@ -93,6 +93,10 @@ body:
description: We only provide support for the most recent version of Portainer and the previous 3 versions. If you are on an older version of Portainer we recommend [upgrading first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
multiple: false
options:
- '2.22.0'
- '2.21.2'
- '2.21.1'
- '2.21.0'
- '2.20.3'
- '2.20.2'
- '2.20.1'

View File

@@ -22,7 +22,6 @@ on:
env:
DOCKER_HUB_REPO: portainerci/portainer-ce
EXTENSION_HUB_REPO: portainerci/portainer-docker-extension
GO_VERSION: 1.21.9
NODE_VERSION: 18.x
jobs:
@@ -34,7 +33,6 @@ jobs:
- { platform: linux, arch: arm64, version: "" }
- { platform: linux, arch: arm, version: "" }
- { platform: linux, arch: ppc64le, version: "" }
- { platform: linux, arch: s390x, version: "" }
- { platform: windows, arch: amd64, version: 1809 }
- { platform: windows, arch: amd64, version: ltsc2022 }
runs-on: ubuntu-latest
@@ -47,7 +45,7 @@ jobs:
- name: '[preparation] set up golang'
uses: actions/setup-go@v5.0.0
with:
go-version: ${{ env.GO_VERSION }}
go-version-file: go.mod
- name: '[preparation] set up node.js'
uses: actions/setup-node@v4.0.1
with:
@@ -103,14 +101,14 @@ jobs:
run: |
if [ "${{ matrix.config.platform }}" == "windows" ]; then
mv dist/portainer dist/portainer.exe
docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} --build-arg OSVERSION=${{ matrix.config.version }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
docker buildx build --output=type=registry --attest type=provenance,mode=max --attest type=sbom,disabled=false --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} --build-arg OSVERSION=${{ matrix.config.version }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
else
docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
docker buildx build --output=type=registry --attest type=provenance,mode=max --attest type=sbom,disabled=false --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
docker buildx build --output=type=registry --attest type=provenance,mode=max --attest type=sbom,disabled=false --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
docker buildx build --output=type=registry --attest type=provenance,mode=max --attest type=sbom,disabled=false --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
docker buildx build --output=type=registry --attest type=provenance,mode=max --attest type=sbom,disabled=false --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
fi
fi
env:
@@ -152,25 +150,17 @@ jobs:
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-s390x" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windows1809-amd64" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windowsltsc2022-amd64"
docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64-alpine" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64-alpine" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine"
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine" \
"${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le-alpine"
if [[ "${GITHUB_REF_NAME}" =~ ^release/.*$ ]]; then
docker buildx imagetools create -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-ppc64le" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-s390x"
docker buildx imagetools create -t "${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64-alpine" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64-alpine" \
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm-alpine"
"${EXTENSION_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64"
fi

View File

@@ -18,7 +18,7 @@ on:
- ready_for_review
env:
GO_VERSION: 1.21.9
GO_VERSION: 1.22.5
NODE_VERSION: 18.x
jobs:
@@ -51,5 +51,5 @@ jobs:
- name: GolangCI-Lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.55.2
version: v1.59.1
args: --timeout=10m -c .golangci.yaml

View File

@@ -6,7 +6,9 @@ on:
workflow_dispatch:
env:
GO_VERSION: 1.21.9
GO_VERSION: 1.22.5
DOCKER_HUB_REPO: portainerci/portainer-ce
DOCKER_HUB_IMAGE_TAG: develop
jobs:
client-dependencies:
@@ -112,7 +114,7 @@ jobs:
uses: docker://docker.io/aquasec/trivy:latest
continue-on-error: true
with:
args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress portainerci/portainer:develop
args: image --ignore-unfixed=true --vuln-type="os,library" --exit-code=1 --format="json" --output="image-trivy.json" --no-progress ${{ env.DOCKER_HUB_REPO }}:${{ env.DOCKER_HUB_IMAGE_TAG }}
- name: upload Trivy image security scan result as artifact
uses: actions/upload-artifact@v3
@@ -141,7 +143,7 @@ jobs:
continue-on-error: true
with:
command: cves
image: portainerci/portainer:develop
image: ${{ env.DOCKER_HUB_REPO }}:${{ env.DOCKER_HUB_IMAGE_TAG }}
sarif-file: image-docker-scout.json
dockerhub-user: ${{ secrets.DOCKER_HUB_USERNAME }}
dockerhub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}

View File

@@ -14,7 +14,7 @@ on:
- '.github/workflows/pr-security.yml'
env:
GO_VERSION: 1.21.9
GO_VERSION: 1.22.5
NODE_VERSION: 18.x
jobs:

View File

@@ -1,10 +1,11 @@
name: Test
env:
GO_VERSION: 1.21.9
GO_VERSION: 1.22.5
NODE_VERSION: 18.x
on:
workflow_dispatch:
pull_request:
branches:
- master
@@ -27,15 +28,22 @@ jobs:
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v2
- uses: actions/setup-node@v2
- name: 'checkout the current branch'
uses: actions/checkout@v4.1.1
with:
ref: ${{ github.event.inputs.branch }}
- name: 'set up node.js'
uses: actions/setup-node@v4.0.1
with:
node-version: ${{ env.NODE_VERSION }}
cache: 'yarn'
- run: yarn --frozen-lockfile
- name: Run tests
run: make test-client ARGS="--maxWorkers=2 --minWorkers=1"
test-server:
strategy:
matrix:
@@ -48,9 +56,21 @@ jobs:
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
- name: 'checkout the current branch'
uses: actions/checkout@v4.1.1
with:
ref: ${{ github.event.inputs.branch }}
- name: 'set up golang'
uses: actions/setup-go@v5.0.0
with:
go-version: ${{ env.GO_VERSION }}
- name: Run tests
- name: 'install dependencies'
run: make test-deps PLATFORM=linux ARCH=amd64
- name: 'update $PATH'
run: echo "$(pwd)/dist" >> $GITHUB_PATH
- name: 'run tests'
run: make test-server

View File

@@ -13,7 +13,7 @@ on:
- ready_for_review
env:
GO_VERSION: 1.21.9
GO_VERSION: 1.22.5
NODE_VERSION: 18.x
jobs:

View File

@@ -9,7 +9,6 @@ linters:
- gosimple
- govet
- errorlint
- exportloopref
linters-settings:
depguard:

View File

@@ -30,7 +30,7 @@ build-server: init-dist ## Build the server binary
./build/build_binary.sh "$(PLATFORM)" "$(ARCH)"
build-image: build-all ## Build the Portainer image locally
docker buildx build --load -t portainerci/portainer:$(TAG) -f build/linux/Dockerfile .
docker buildx build --load -t portainerci/portainer-ce:$(TAG) -f build/linux/Dockerfile .
build-storybook: ## Build and serve the storybook files
yarn storybook:build
@@ -64,6 +64,9 @@ clean: ## Remove all build and download artifacts
.PHONY: test test-client test-server
test: test-server test-client ## Run all tests
test-deps: init-dist
./build/download_docker_compose_binary.sh $(PLATFORM) $(ARCH) $(shell jq -r '.dockerCompose' < "./binary-version.json")
test-client: ## Run client tests
yarn test $(ARGS)
@@ -82,6 +85,8 @@ dev-client: ## Run the client in development mode
dev-server: build-server ## Run the server in development mode
@./dev/run_container.sh
dev-server-podman: build-server ## Run the server in development mode
@./dev/run_container_podman.sh
##@ Format
.PHONY: format format-client format-server

View File

@@ -10,7 +10,7 @@ import (
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/internal/url"
"github.com/portainer/portainer/api/url"
)
// GetAgentVersionAndPlatform returns the agent version and platform

View File

@@ -3,7 +3,6 @@ package apikey
import (
"testing"
"github.com/portainer/portainer/api/internal/securecookie"
"github.com/stretchr/testify/assert"
)
@@ -34,17 +33,19 @@ func Test_generateRandomKey(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := securecookie.GenerateRandomKey(tt.wantLenth)
got := GenerateRandomKey(tt.wantLenth)
is.Equal(tt.wantLenth, len(got))
})
}
t.Run("Generated keys are unique", func(t *testing.T) {
keys := make(map[string]bool)
for i := 0; i < 100; i++ {
key := securecookie.GenerateRandomKey(8)
for range 100 {
key := GenerateRandomKey(8)
_, ok := keys[string(key)]
is.False(ok)
keys[string(key)] = true
}
})

View File

@@ -1,69 +1,79 @@
package apikey
import (
lru "github.com/hashicorp/golang-lru"
portainer "github.com/portainer/portainer/api"
lru "github.com/hashicorp/golang-lru"
)
const defaultAPIKeyCacheSize = 1024
const DefaultAPIKeyCacheSize = 1024
// entry is a tuple containing the user and API key associated to an API key digest
type entry struct {
user portainer.User
type entry[T any] struct {
user T
apiKey portainer.APIKey
}
// apiKeyCache is a concurrency-safe, in-memory cache which primarily exists for to reduce database roundtrips.
type UserCompareFn[T any] func(T, portainer.UserID) bool
// ApiKeyCache is a concurrency-safe, in-memory cache which primarily exists for to reduce database roundtrips.
// We store the api-key digest (keys) and the associated user and key-data (values) in the cache.
// This is required because HTTP requests will contain only the api-key digest in the x-api-key request header;
// digest value must be mapped to a portainer user (and respective key data) for validation.
// This cache is used to avoid multiple database queries to retrieve these user/key associated to the digest.
type apiKeyCache struct {
type ApiKeyCache[T any] struct {
// cache type [string]entry cache (key: string(digest), value: user/key entry)
// note: []byte keys are not supported by golang-lru Cache
cache *lru.Cache
cache *lru.Cache
userCmpFn UserCompareFn[T]
}
// NewAPIKeyCache creates a new cache for API keys
func NewAPIKeyCache(cacheSize int) *apiKeyCache {
func NewAPIKeyCache[T any](cacheSize int, userCompareFn UserCompareFn[T]) *ApiKeyCache[T] {
cache, _ := lru.New(cacheSize)
return &apiKeyCache{cache: cache}
return &ApiKeyCache[T]{cache: cache, userCmpFn: userCompareFn}
}
// Get returns the user/key associated to an api-key's digest
// This is required because HTTP requests will contain the digest of the API key in header,
// the digest value must be mapped to a portainer user.
func (c *apiKeyCache) Get(digest string) (portainer.User, portainer.APIKey, bool) {
func (c *ApiKeyCache[T]) Get(digest string) (T, portainer.APIKey, bool) {
val, ok := c.cache.Get(digest)
if !ok {
return portainer.User{}, portainer.APIKey{}, false
var t T
return t, portainer.APIKey{}, false
}
tuple := val.(entry)
tuple := val.(entry[T])
return tuple.user, tuple.apiKey, true
}
// Set persists a user/key entry to the cache
func (c *apiKeyCache) Set(digest string, user portainer.User, apiKey portainer.APIKey) {
c.cache.Add(digest, entry{
func (c *ApiKeyCache[T]) Set(digest string, user T, apiKey portainer.APIKey) {
c.cache.Add(digest, entry[T]{
user: user,
apiKey: apiKey,
})
}
// Delete evicts a digest's user/key entry key from the cache
func (c *apiKeyCache) Delete(digest string) {
func (c *ApiKeyCache[T]) Delete(digest string) {
c.cache.Remove(digest)
}
// InvalidateUserKeyCache loops through all the api-keys associated to a user and removes them from the cache
func (c *apiKeyCache) InvalidateUserKeyCache(userId portainer.UserID) bool {
func (c *ApiKeyCache[T]) InvalidateUserKeyCache(userId portainer.UserID) bool {
present := false
for _, k := range c.cache.Keys() {
user, _, _ := c.Get(k.(string))
if user.ID == userId {
if c.userCmpFn(user, userId) {
present = c.cache.Remove(k)
}
}
return present
}

View File

@@ -10,11 +10,11 @@ import (
func Test_apiKeyCacheGet(t *testing.T) {
is := assert.New(t)
keyCache := NewAPIKeyCache(10)
keyCache := NewAPIKeyCache(10, compareUser)
// pre-populate cache
keyCache.cache.Add(string("foo"), entry{user: portainer.User{}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string(""), entry{user: portainer.User{}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string("foo"), entry[portainer.User]{user: portainer.User{}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string(""), entry[portainer.User]{user: portainer.User{}, apiKey: portainer.APIKey{}})
tests := []struct {
digest string
@@ -45,7 +45,7 @@ func Test_apiKeyCacheGet(t *testing.T) {
func Test_apiKeyCacheSet(t *testing.T) {
is := assert.New(t)
keyCache := NewAPIKeyCache(10)
keyCache := NewAPIKeyCache(10, compareUser)
// pre-populate cache
keyCache.Set("bar", portainer.User{ID: 2}, portainer.APIKey{})
@@ -57,23 +57,23 @@ func Test_apiKeyCacheSet(t *testing.T) {
val, ok := keyCache.cache.Get(string("bar"))
is.True(ok)
tuple := val.(entry)
tuple := val.(entry[portainer.User])
is.Equal(portainer.User{ID: 2}, tuple.user)
val, ok = keyCache.cache.Get(string("foo"))
is.True(ok)
tuple = val.(entry)
tuple = val.(entry[portainer.User])
is.Equal(portainer.User{ID: 3}, tuple.user)
}
func Test_apiKeyCacheDelete(t *testing.T) {
is := assert.New(t)
keyCache := NewAPIKeyCache(10)
keyCache := NewAPIKeyCache(10, compareUser)
t.Run("Delete an existing entry", func(t *testing.T) {
keyCache.cache.Add(string("foo"), entry{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string("foo"), entry[portainer.User]{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
keyCache.Delete("foo")
_, ok := keyCache.cache.Get(string("foo"))
@@ -128,7 +128,7 @@ func Test_apiKeyCacheLRU(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
keyCache := NewAPIKeyCache(test.cacheLen)
keyCache := NewAPIKeyCache(test.cacheLen, compareUser)
for _, key := range test.key {
keyCache.Set(key, portainer.User{ID: 1}, portainer.APIKey{})
@@ -150,10 +150,10 @@ func Test_apiKeyCacheLRU(t *testing.T) {
func Test_apiKeyCacheInvalidateUserKeyCache(t *testing.T) {
is := assert.New(t)
keyCache := NewAPIKeyCache(10)
keyCache := NewAPIKeyCache(10, compareUser)
t.Run("Removes users keys from cache", func(t *testing.T) {
keyCache.cache.Add(string("foo"), entry{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string("foo"), entry[portainer.User]{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
ok := keyCache.InvalidateUserKeyCache(1)
is.True(ok)
@@ -163,8 +163,8 @@ func Test_apiKeyCacheInvalidateUserKeyCache(t *testing.T) {
})
t.Run("Does not affect other keys", func(t *testing.T) {
keyCache.cache.Add(string("foo"), entry{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string("bar"), entry{user: portainer.User{ID: 2}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string("foo"), entry[portainer.User]{user: portainer.User{ID: 1}, apiKey: portainer.APIKey{}})
keyCache.cache.Add(string("bar"), entry[portainer.User]{user: portainer.User{ID: 2}, apiKey: portainer.APIKey{}})
ok := keyCache.InvalidateUserKeyCache(1)
is.True(ok)

View File

@@ -1,14 +1,15 @@
package apikey
import (
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"fmt"
"io"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/internal/securecookie"
"github.com/pkg/errors"
)
@@ -20,30 +21,45 @@ var ErrInvalidAPIKey = errors.New("Invalid API key")
type apiKeyService struct {
apiKeyRepository dataservices.APIKeyRepository
userRepository dataservices.UserService
cache *apiKeyCache
cache *ApiKeyCache[portainer.User]
}
// GenerateRandomKey generates a random key of specified length
// source: https://github.com/gorilla/securecookie/blob/master/securecookie.go#L515
func GenerateRandomKey(length int) []byte {
k := make([]byte, length)
if _, err := io.ReadFull(rand.Reader, k); err != nil {
return nil
}
return k
}
func compareUser(u portainer.User, id portainer.UserID) bool {
return u.ID == id
}
func NewAPIKeyService(apiKeyRepository dataservices.APIKeyRepository, userRepository dataservices.UserService) *apiKeyService {
return &apiKeyService{
apiKeyRepository: apiKeyRepository,
userRepository: userRepository,
cache: NewAPIKeyCache(defaultAPIKeyCacheSize),
cache: NewAPIKeyCache(DefaultAPIKeyCacheSize, compareUser),
}
}
// HashRaw computes a hash digest of provided raw API key.
func (a *apiKeyService) HashRaw(rawKey string) string {
hashDigest := sha256.Sum256([]byte(rawKey))
return base64.StdEncoding.EncodeToString(hashDigest[:])
}
// GenerateApiKey generates a raw API key for a user (for one-time display).
// The generated API key is stored in the cache and database.
func (a *apiKeyService) GenerateApiKey(user portainer.User, description string) (string, *portainer.APIKey, error) {
randKey := securecookie.GenerateRandomKey(32)
randKey := GenerateRandomKey(32)
encodedRawAPIKey := base64.StdEncoding.EncodeToString(randKey)
prefixedAPIKey := portainerAPIKeyPrefix + encodedRawAPIKey
hashDigest := a.HashRaw(prefixedAPIKey)
apiKey := &portainer.APIKey{
@@ -54,8 +70,7 @@ func (a *apiKeyService) GenerateApiKey(user portainer.User, description string)
Digest: hashDigest,
}
err := a.apiKeyRepository.Create(apiKey)
if err != nil {
if err := a.apiKeyRepository.Create(apiKey); err != nil {
return "", nil, errors.Wrap(err, "Unable to create API key")
}
@@ -78,7 +93,6 @@ func (a *apiKeyService) GetAPIKeys(userID portainer.UserID) ([]portainer.APIKey,
// GetDigestUserAndKey returns the user and api-key associated to a specified hash digest.
// A cache lookup is performed first; if the user/api-key is not found in the cache, respective database lookups are performed.
func (a *apiKeyService) GetDigestUserAndKey(digest string) (portainer.User, portainer.APIKey, error) {
// get api key from cache if possible
cachedUser, cachedKey, ok := a.cache.Get(digest)
if ok {
return cachedUser, cachedKey, nil
@@ -106,20 +120,21 @@ func (a *apiKeyService) UpdateAPIKey(apiKey *portainer.APIKey) error {
if err != nil {
return errors.Wrap(err, "Unable to retrieve API key")
}
a.cache.Set(apiKey.Digest, user, *apiKey)
return a.apiKeyRepository.Update(apiKey.ID, apiKey)
}
// DeleteAPIKey deletes an API key and removes the digest/api-key entry from the cache.
func (a *apiKeyService) DeleteAPIKey(apiKeyID portainer.APIKeyID) error {
// get api-key digest to remove from cache
apiKey, err := a.apiKeyRepository.Read(apiKeyID)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Unable to retrieve API key: %d", apiKeyID))
}
// delete the user/api-key from cache
a.cache.Delete(apiKey.Digest)
return a.apiKeyRepository.Delete(apiKeyID)
}

View File

@@ -15,8 +15,10 @@ import (
func TestPingAgentPanic(t *testing.T) {
endpoint := &portainer.Endpoint{
ID: 1,
Type: portainer.EdgeAgentOnDockerEnvironment,
ID: 1,
EdgeID: "test-edge-id",
Type: portainer.EdgeAgentOnDockerEnvironment,
UserTrusted: true,
}
_, store := datastore.MustNewTestStore(t, true, true)
@@ -42,7 +44,8 @@ func TestPingAgentPanic(t *testing.T) {
errCh <- srv.Serve(ln)
}()
s.Open(endpoint)
err = s.Open(endpoint)
require.NoError(t, err)
s.activeTunnels[endpoint.ID].Port = ln.Addr().(*net.TCPAddr).Port
require.Error(t, s.pingAgent(endpoint.ID))

View File

@@ -24,14 +24,24 @@ const (
maxAvailablePort = 65535
)
var (
ErrNonEdgeEnv = errors.New("cannot open a tunnel for non-edge environments")
ErrAsyncEnv = errors.New("cannot open a tunnel for async edge environments")
ErrInvalidEnv = errors.New("cannot open a tunnel for an invalid environment")
)
// Open will mark the tunnel as REQUIRED so the agent opens it
func (s *Service) Open(endpoint *portainer.Endpoint) error {
if !endpointutils.IsEdgeEndpoint(endpoint) {
return errors.New("cannot open a tunnel for non-edge environments")
return ErrNonEdgeEnv
}
if endpoint.Edge.AsyncMode {
return errors.New("cannot open a tunnel for async edge environments")
return ErrAsyncEnv
}
if endpoint.ID == 0 || endpoint.EdgeID == "" || !endpoint.UserTrusted {
return ErrInvalidEnv
}
s.mu.Lock()

View File

@@ -17,17 +17,14 @@ import (
type Service struct{}
var (
errInvalidEndpointProtocol = errors.New("Invalid environment protocol: Portainer only supports unix://, npipe:// or tcp://")
errSocketOrNamedPipeNotFound = errors.New("Unable to locate Unix socket or named pipe")
errInvalidSnapshotInterval = errors.New("Invalid snapshot interval")
errAdminPassExcludeAdminPassFile = errors.New("Cannot use --admin-password with --admin-password-file")
ErrInvalidEndpointProtocol = errors.New("Invalid environment protocol: Portainer only supports unix://, npipe:// or tcp://")
ErrSocketOrNamedPipeNotFound = errors.New("Unable to locate Unix socket or named pipe")
ErrInvalidSnapshotInterval = errors.New("Invalid snapshot interval")
ErrAdminPassExcludeAdminPassFile = errors.New("Cannot use --admin-password with --admin-password-file")
)
// ParseFlags parse the CLI flags and return a portainer.Flags struct
func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
kingpin.Version(version)
flags := &portainer.CLIFlags{
func CLIFlags() *portainer.CLIFlags {
return &portainer.CLIFlags{
Addr: kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(),
AddrHTTPS: kingpin.Flag("bind-https", "Address and port to serve Portainer via https").Default(defaultHTTPSBindAddress).String(),
TunnelAddr: kingpin.Flag("tunnel-addr", "Address to serve the tunnel server").Default(defaultTunnelServerAddress).String(),
@@ -63,6 +60,13 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
LogLevel: kingpin.Flag("log-level", "Set the minimum logging level to show").Default("INFO").Enum("DEBUG", "INFO", "WARN", "ERROR"),
LogMode: kingpin.Flag("log-mode", "Set the logging output mode").Default("PRETTY").Enum("NOCOLOR", "PRETTY", "JSON"),
}
}
// ParseFlags parse the CLI flags and return a portainer.Flags struct
func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
kingpin.Version(version)
flags := CLIFlags()
kingpin.Parse()
@@ -82,18 +86,16 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
displayDeprecationWarnings(flags)
err := validateEndpointURL(*flags.EndpointURL)
if err != nil {
if err := validateEndpointURL(*flags.EndpointURL); err != nil {
return err
}
err = validateSnapshotInterval(*flags.SnapshotInterval)
if err != nil {
if err := validateSnapshotInterval(*flags.SnapshotInterval); err != nil {
return err
}
if *flags.AdminPassword != "" && *flags.AdminPasswordFile != "" {
return errAdminPassExcludeAdminPassFile
return ErrAdminPassExcludeAdminPassFile
}
return nil
@@ -115,15 +117,16 @@ func validateEndpointURL(endpointURL string) error {
}
if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") && !strings.HasPrefix(endpointURL, "npipe://") {
return errInvalidEndpointProtocol
return ErrInvalidEndpointProtocol
}
if strings.HasPrefix(endpointURL, "unix://") || strings.HasPrefix(endpointURL, "npipe://") {
socketPath := strings.TrimPrefix(endpointURL, "unix://")
socketPath = strings.TrimPrefix(socketPath, "npipe://")
if _, err := os.Stat(socketPath); err != nil {
if os.IsNotExist(err) {
return errSocketOrNamedPipeNotFound
return ErrSocketOrNamedPipeNotFound
}
return err
@@ -138,9 +141,8 @@ func validateSnapshotInterval(snapshotInterval string) error {
return nil
}
_, err := time.ParseDuration(snapshotInterval)
if err != nil {
return errInvalidSnapshotInterval
if _, err := time.ParseDuration(snapshotInterval); err != nil {
return ErrInvalidSnapshotInterval
}
return nil

View File

@@ -54,7 +54,7 @@ func setLoggingMode(mode string) {
}
}
func formatMessage(i interface{}) string {
func formatMessage(i any) string {
if i == nil {
return ""
}

View File

@@ -1,6 +1,7 @@
package main
import (
"cmp"
"context"
"crypto/sha256"
"os"
@@ -44,6 +45,7 @@ import (
"github.com/portainer/portainer/api/pendingactions"
"github.com/portainer/portainer/api/pendingactions/actions"
"github.com/portainer/portainer/api/pendingactions/handlers"
"github.com/portainer/portainer/api/platform"
"github.com/portainer/portainer/api/scheduler"
"github.com/portainer/portainer/api/stacks/deployments"
"github.com/portainer/portainer/pkg/featureflags"
@@ -56,14 +58,14 @@ import (
)
func initCLI() *portainer.CLIFlags {
var cliService portainer.CLIService = &cli.Service{}
cliService := &cli.Service{}
flags, err := cliService.ParseFlags(portainer.APIVersion)
if err != nil {
log.Fatal().Err(err).Msg("failed parsing flags")
}
err = cliService.ValidateFlags(flags)
if err != nil {
if err := cliService.ValidateFlags(flags); err != nil {
log.Fatal().Err(err).Msg("failed validating flags")
}
@@ -94,14 +96,14 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
}
store := datastore.NewStore(*flags.Data, fileService, connection)
isNew, err := store.Open()
if err != nil {
log.Fatal().Err(err).Msg("failed opening store")
}
if *flags.Rollback {
err := store.Rollback(false)
if err != nil {
if err := store.Rollback(false); err != nil {
log.Fatal().Err(err).Msg("failed rolling back")
}
@@ -110,8 +112,7 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
}
// Init sets some defaults - it's basically a migration
err = store.Init()
if err != nil {
if err := store.Init(); err != nil {
log.Fatal().Err(err).Msg("failed initializing data store")
}
@@ -133,25 +134,23 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
}
store.VersionService.UpdateVersion(&v)
err = updateSettingsFromFlags(store, flags)
if err != nil {
if err := updateSettingsFromFlags(store, flags); err != nil {
log.Fatal().Err(err).Msg("failed updating settings from flags")
}
} else {
err = store.MigrateData()
if err != nil {
if err := store.MigrateData(); err != nil {
log.Fatal().Err(err).Msg("failed migration")
}
}
err = updateSettingsFromFlags(store, flags)
if err != nil {
if err := updateSettingsFromFlags(store, flags); err != nil {
log.Fatal().Err(err).Msg("failed updating settings from flags")
}
// this is for the db restore functionality - needs more tests.
go func() {
<-shutdownCtx.Done()
defer connection.Close()
}()
@@ -205,36 +204,16 @@ func initJWTService(userSessionTimeout string, dataStore dataservices.DataStore)
userSessionTimeout = portainer.DefaultUserSessionTimeout
}
jwtService, err := jwt.NewService(userSessionTimeout, dataStore)
if err != nil {
return nil, err
}
return jwtService, nil
return jwt.NewService(userSessionTimeout, dataStore)
}
func initDigitalSignatureService() portainer.DigitalSignatureService {
return crypto.NewECDSAService(os.Getenv("AGENT_SECRET"))
}
func initCryptoService() portainer.CryptoService {
return &crypto.Service{}
}
func initLDAPService() portainer.LDAPService {
return &ldap.Service{}
}
func initOAuthService() portainer.OAuthService {
return oauth.NewService()
}
func initGitService(ctx context.Context) portainer.GitService {
return git.NewService(ctx)
}
func initSSLService(addr, certPath, keyPath string, fileService portainer.FileService, dataStore dataservices.DataStore, shutdownTrigger context.CancelFunc) (*ssl.Service, error) {
slices := strings.Split(addr, ":")
host := slices[0]
if host == "" {
host = "0.0.0.0"
@@ -242,22 +221,13 @@ func initSSLService(addr, certPath, keyPath string, fileService portainer.FileSe
sslService := ssl.NewService(fileService, dataStore, shutdownTrigger)
err := sslService.Init(host, certPath, keyPath)
if err != nil {
if err := sslService.Init(host, certPath, keyPath); err != nil {
return nil, err
}
return sslService, nil
}
func initDockerClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService) *dockerclient.ClientFactory {
return dockerclient.NewClientFactory(signatureService, reverseTunnelService)
}
func initKubernetesClientFactory(signatureService portainer.DigitalSignatureService, reverseTunnelService portainer.ReverseTunnelService, dataStore dataservices.DataStore, instanceID, addrHTTPS, userSessionTimeout string) (*kubecli.ClientFactory, error) {
return kubecli.NewClientFactory(signatureService, reverseTunnelService, dataStore, instanceID, addrHTTPS, userSessionTimeout)
}
func initSnapshotService(
snapshotIntervalFromFlag string,
dataStore dataservices.DataStore,
@@ -290,34 +260,21 @@ func updateSettingsFromFlags(dataStore dataservices.DataStore, flags *portainer.
return err
}
if *flags.SnapshotInterval != "" {
settings.SnapshotInterval = *flags.SnapshotInterval
}
if *flags.Logo != "" {
settings.LogoURL = *flags.Logo
}
if *flags.EnableEdgeComputeFeatures {
settings.EnableEdgeComputeFeatures = *flags.EnableEdgeComputeFeatures
}
if *flags.Templates != "" {
settings.TemplatesURL = *flags.Templates
}
settings.SnapshotInterval = *cmp.Or(flags.SnapshotInterval, &settings.SnapshotInterval)
settings.LogoURL = *cmp.Or(flags.Logo, &settings.LogoURL)
settings.EnableEdgeComputeFeatures = *cmp.Or(flags.EnableEdgeComputeFeatures, &settings.EnableEdgeComputeFeatures)
settings.TemplatesURL = *cmp.Or(flags.Templates, &settings.TemplatesURL)
if *flags.Labels != nil {
settings.BlackListedLabels = *flags.Labels
}
settings.AgentSecret = ""
if agentKey, ok := os.LookupEnv("AGENT_SECRET"); ok {
settings.AgentSecret = agentKey
} else {
settings.AgentSecret = ""
}
err = dataStore.Settings().UpdateSettings(settings)
if err != nil {
if err := dataStore.Settings().UpdateSettings(settings); err != nil {
return err
}
@@ -340,6 +297,7 @@ func loadAndParseKeyPair(fileService portainer.FileService, signatureService por
if err != nil {
return err
}
return signatureService.ParseKeyPair(private, public)
}
@@ -348,7 +306,9 @@ func generateAndStoreKeyPair(fileService portainer.FileService, signatureService
if err != nil {
return err
}
privateHeader, publicHeader := signatureService.PEMHeaders()
return fileService.StoreKeyPair(private, public, privateHeader, publicHeader)
}
@@ -361,6 +321,7 @@ func initKeyPair(fileService portainer.FileService, signatureService portainer.D
if existingKeyPair {
return loadAndParseKeyPair(fileService, signatureService)
}
return generateAndStoreKeyPair(fileService, signatureService)
}
@@ -378,6 +339,7 @@ func loadEncryptionSecretKey(keyfilename string) []byte {
// return a 32 byte hash of the secret (required for AES)
hash := sha256.Sum256(content)
return hash[:]
}
@@ -422,17 +384,17 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
log.Fatal().Err(err).Msg("failed initializing JWT service")
}
ldapService := initLDAPService()
ldapService := &ldap.Service{}
oauthService := initOAuthService()
oauthService := oauth.NewService()
gitService := initGitService(shutdownCtx)
gitService := git.NewService(shutdownCtx)
openAMTService := openamt.NewService()
cryptoService := initCryptoService()
cryptoService := &crypto.Service{}
digitalSignatureService := initDigitalSignatureService()
signatureService := initDigitalSignatureService()
edgeStacksService := edgestacks.NewService(dataStore)
@@ -446,15 +408,18 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
log.Fatal().Err(err).Msg("failed to get SSL settings")
}
err = initKeyPair(fileService, digitalSignatureService)
if err != nil {
if err := initKeyPair(fileService, signatureService); err != nil {
log.Fatal().Err(err).Msg("failed initializing key pair")
}
reverseTunnelService := chisel.NewService(dataStore, shutdownCtx, fileService)
dockerClientFactory := initDockerClientFactory(digitalSignatureService, reverseTunnelService)
kubernetesClientFactory, err := initKubernetesClientFactory(digitalSignatureService, reverseTunnelService, dataStore, instanceID, *flags.AddrHTTPS, settings.UserSessionTimeout)
dockerClientFactory := dockerclient.NewClientFactory(signatureService, reverseTunnelService)
kubernetesClientFactory, err := kubecli.NewClientFactory(signatureService, reverseTunnelService, dataStore, instanceID, *flags.AddrHTTPS, settings.UserSessionTimeout)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing Kubernetes Client Factory service")
}
authorizationService := authorization.NewService(dataStore)
authorizationService.K8sClientFactory = kubernetesClientFactory
@@ -476,12 +441,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
composeStackManager := initComposeStackManager(composeDeployer, proxyManager)
swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore)
swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, signatureService, fileService, reverseTunnelService, dataStore)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing swarm stack manager")
}
kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, signatureService, proxyManager, *flags.Assets)
pendingActionsService := pendingactions.NewService(dataStore, kubernetesClientFactory)
pendingActionsService.RegisterHandler(actions.CleanNAPWithOverridePolicies, handlers.NewHandlerCleanNAPWithOverridePolicies(authorizationService, dataStore))
@@ -492,17 +457,17 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
if err != nil {
log.Fatal().Err(err).Msg("failed initializing snapshot service")
}
snapshotService.Start()
proxyManager.NewProxyFactory(dataStore, digitalSignatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService, snapshotService)
proxyManager.NewProxyFactory(dataStore, signatureService, reverseTunnelService, dockerClientFactory, kubernetesClientFactory, kubernetesTokenCacheManager, gitService, snapshotService)
helmPackageManager, err := initHelmPackageManager(*flags.Assets)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing helm package manager")
}
err = edge.LoadEdgeJobs(dataStore, reverseTunnelService)
if err != nil {
if err := edge.LoadEdgeJobs(dataStore, reverseTunnelService); err != nil {
log.Fatal().Err(err).Msg("failed loading edge jobs from database")
}
@@ -514,6 +479,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
go endpointutils.InitEndpoint(shutdownCtx, adminCreationDone, flags, dataStore, snapshotService)
adminPasswordHash := ""
if *flags.AdminPasswordFile != "" {
content, err := fileService.GetFileContent(*flags.AdminPasswordFile, "")
if err != nil {
@@ -536,14 +502,14 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
if len(users) == 0 {
log.Info().Msg("created admin user with the given password.")
user := &portainer.User{
Username: "admin",
Role: portainer.AdministratorRole,
Password: adminPasswordHash,
}
err := dataStore.User().Create(user)
if err != nil {
if err := dataStore.User().Create(user); err != nil {
log.Fatal().Err(err).Msg("failed creating admin user")
}
@@ -554,8 +520,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
}
}
err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
if err != nil {
if err := reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService); err != nil {
log.Fatal().Err(err).Msg("failed starting tunnel server")
}
@@ -568,7 +533,20 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
log.Fatal().Msg("failed to fetch SSL settings from DB")
}
upgradeService, err := upgrade.NewService(*flags.Assets, composeDeployer, kubernetesClientFactory)
platformService, err := platform.NewService(dataStore)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing platform service")
}
upgradeService, err := upgrade.NewService(
*flags.Assets,
kubernetesClientFactory,
dockerClientFactory,
composeStackManager,
dataStore,
fileService,
stackDeployer,
)
if err != nil {
log.Fatal().Err(err).Msg("failed initializing upgrade service")
}
@@ -613,7 +591,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
ProxyManager: proxyManager,
KubernetesTokenCacheManager: kubernetesTokenCacheManager,
KubeClusterAccessService: kubeClusterAccessService,
SignatureService: digitalSignatureService,
SignatureService: signatureService,
SnapshotService: snapshotService,
SSLService: sslService,
DockerClientFactory: dockerClientFactory,
@@ -625,6 +603,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
UpgradeService: upgradeService,
AdminCreationDone: adminCreationDone,
PendingActionsService: pendingActionsService,
PlatformService: platformService,
}
}
@@ -639,6 +618,7 @@ func main() {
for {
server := buildServer(flags)
log.Info().
Str("version", portainer.APIVersion).
Str("build_number", build.BuildNumber).

View File

@@ -17,11 +17,11 @@ func (kcl *KubeClient) GetConfigMapsAndSecrets(namespace string) ([]models.K8sCo
// use closures to capture the current kube client and namespace by declaring wrapper functions
// that match the interface signature for concurrent.Func
listConfigMaps := func(ctx context.Context) (interface{}, error) {
listConfigMaps := func(ctx context.Context) (any, error) {
return kcl.cli.CoreV1().ConfigMaps(namespace).List(context.Background(), meta.ListOptions{})
}
listSecrets := func(ctx context.Context) (interface{}, error) {
listSecrets := func(ctx context.Context) (any, error) {
return kcl.cli.CoreV1().Secrets(namespace).List(context.Background(), meta.ListOptions{})
}
@@ -93,6 +93,7 @@ type Func func(ctx context.Context) (any, error)
// Run runs a list of functions returns the results
func Run(ctx context.Context, maxConcurrency int, tasks ...Func) ([]Result, error) {
var wg sync.WaitGroup
resultsChan := make(chan Result, len(tasks))
taskChan := make(chan Func, len(tasks))
@@ -101,6 +102,7 @@ func Run(ctx context.Context, maxConcurrency int, tasks ...Func) ([]Result, erro
runTask := func() {
defer wg.Done()
for fn := range taskChan {
result, err := fn(localCtx)
resultsChan <- Result{Result: result, Err: err}
@@ -113,7 +115,7 @@ func Run(ctx context.Context, maxConcurrency int, tasks ...Func) ([]Result, erro
}
// Start worker goroutines
for i := 0; i < maxConcurrency; i++ {
for range maxConcurrency {
wg.Add(1)
go runTask()
}
@@ -135,8 +137,10 @@ func Run(ctx context.Context, maxConcurrency int, tasks ...Func) ([]Result, erro
for r := range resultsChan {
if r.Err != nil {
cancelCtx()
return nil, r.Err
}
results = append(results, r)
}

View File

@@ -5,22 +5,21 @@ import (
)
type ReadTransaction interface {
GetObject(bucketName string, key []byte, object interface{}) error
GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error
GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error
GetObject(bucketName string, key []byte, object any) error
GetAll(bucketName string, obj any, append func(o any) (any, error)) error
GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj any, append func(o any) (any, error)) error
}
type Transaction interface {
ReadTransaction
SetServiceName(bucketName string) error
UpdateObject(bucketName string, key []byte, object interface{}) error
UpdateObject(bucketName string, key []byte, object any) error
DeleteObject(bucketName string, key []byte) error
CreateObject(bucketName string, fn func(uint64) (int, interface{})) error
CreateObjectWithId(bucketName string, id int, obj interface{}) error
CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error
DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error
CreateObject(bucketName string, fn func(uint64) (int, any)) error
CreateObjectWithId(bucketName string, id int, obj any) error
CreateObjectWithStringId(bucketName string, id []byte, obj any) error
DeleteAllObjects(bucketName string, obj any, matching func(o any) (id int, ok bool)) error
GetNextIdentifier(bucketName string) int
}
@@ -46,8 +45,8 @@ type Connection interface {
NeedsEncryptionMigration() (bool, error)
SetEncrypted(encrypted bool)
BackupMetadata() (map[string]interface{}, error)
RestoreMetadata(s map[string]interface{}) error
BackupMetadata() (map[string]any, error)
RestoreMetadata(s map[string]any) error
UpdateObjectFunc(bucketName string, key []byte, object any, updateFn func()) error
ConvertToKey(v int) []byte

View File

@@ -8,6 +8,7 @@ import (
"math"
"os"
"path"
"strconv"
"time"
portainer "github.com/portainer/portainer/api"
@@ -73,7 +74,6 @@ func (connection *DbConnection) IsEncryptedStore() bool {
// NeedsEncryptionMigration returns true if database encryption is enabled and
// we have an un-encrypted DB that requires migration to an encrypted DB
func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
// Cases: Note, we need to check both portainer.db and portainer.edb
// to determine if it's a new store. We only need to differentiate between cases 2,3 and 5
@@ -121,11 +121,11 @@ func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
// Open opens and initializes the BoltDB database.
func (connection *DbConnection) Open() error {
log.Info().Str("filename", connection.GetDatabaseFileName()).Msg("loading PortainerDB")
// Now we open the db
databasePath := connection.GetDatabaseFilePath()
db, err := bolt.Open(databasePath, 0600, &bolt.Options{
Timeout: 1 * time.Second,
InitialMmapSize: connection.InitialMmapSize,
@@ -178,6 +178,7 @@ func (connection *DbConnection) ViewTx(fn func(portainer.Transaction) error) err
func (connection *DbConnection) BackupTo(w io.Writer) error {
return connection.View(func(tx *bolt.Tx) error {
_, err := tx.WriteTo(w)
return err
})
}
@@ -192,6 +193,7 @@ func (connection *DbConnection) ExportRaw(filename string) error {
if err != nil {
return err
}
return os.WriteFile(filename, b, 0600)
}
@@ -201,6 +203,7 @@ func (connection *DbConnection) ExportRaw(filename string) error {
func (connection *DbConnection) ConvertToKey(v int) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(v))
return b
}
@@ -212,7 +215,7 @@ func keyToString(b []byte) string {
v := binary.BigEndian.Uint64(b)
if v <= math.MaxInt32 {
return fmt.Sprintf("%d", v)
return strconv.FormatUint(v, 10)
}
return string(b)
@@ -226,7 +229,7 @@ func (connection *DbConnection) SetServiceName(bucketName string) error {
}
// GetObject is a generic function used to retrieve an unmarshalled object from a database.
func (connection *DbConnection) GetObject(bucketName string, key []byte, object interface{}) error {
func (connection *DbConnection) GetObject(bucketName string, key []byte, object any) error {
return connection.ViewTx(func(tx portainer.Transaction) error {
return tx.GetObject(bucketName, key, object)
})
@@ -241,7 +244,7 @@ func (connection *DbConnection) getEncryptionKey() []byte {
}
// UpdateObject is a generic function used to update an object inside a database.
func (connection *DbConnection) UpdateObject(bucketName string, key []byte, object interface{}) error {
func (connection *DbConnection) UpdateObject(bucketName string, key []byte, object any) error {
return connection.UpdateTx(func(tx portainer.Transaction) error {
return tx.UpdateObject(bucketName, key, object)
})
@@ -282,7 +285,7 @@ func (connection *DbConnection) DeleteObject(bucketName string, key []byte) erro
// DeleteAllObjects delete all objects where matching() returns (id, ok).
// TODO: think about how to return the error inside (maybe change ok to type err, and use "notfound"?
func (connection *DbConnection) DeleteAllObjects(bucketName string, obj interface{}, matching func(o interface{}) (id int, ok bool)) error {
func (connection *DbConnection) DeleteAllObjects(bucketName string, obj any, matching func(o any) (id int, ok bool)) error {
return connection.UpdateTx(func(tx portainer.Transaction) error {
return tx.DeleteAllObjects(bucketName, obj, matching)
})
@@ -301,71 +304,64 @@ func (connection *DbConnection) GetNextIdentifier(bucketName string) int {
}
// CreateObject creates a new object in the bucket, using the next bucket sequence id
func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64) (int, interface{})) error {
func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64) (int, any)) error {
return connection.UpdateTx(func(tx portainer.Transaction) error {
return tx.CreateObject(bucketName, fn)
})
}
// CreateObjectWithId creates a new object in the bucket, using the specified id
func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, obj interface{}) error {
func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, obj any) error {
return connection.UpdateTx(func(tx portainer.Transaction) error {
return tx.CreateObjectWithId(bucketName, id, obj)
})
}
// CreateObjectWithStringId creates a new object in the bucket, using the specified id
func (connection *DbConnection) CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error {
func (connection *DbConnection) CreateObjectWithStringId(bucketName string, id []byte, obj any) error {
return connection.UpdateTx(func(tx portainer.Transaction) error {
return tx.CreateObjectWithStringId(bucketName, id, obj)
})
}
func (connection *DbConnection) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
func (connection *DbConnection) GetAll(bucketName string, obj any, appendFn func(o any) (any, error)) error {
return connection.ViewTx(func(tx portainer.Transaction) error {
return tx.GetAll(bucketName, obj, append)
return tx.GetAll(bucketName, obj, appendFn)
})
}
// TODO: decide which Unmarshal to use, and use one...
func (connection *DbConnection) GetAllWithJsoniter(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
func (connection *DbConnection) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj any, appendFn func(o any) (any, error)) error {
return connection.ViewTx(func(tx portainer.Transaction) error {
return tx.GetAllWithJsoniter(bucketName, obj, append)
})
}
func (connection *DbConnection) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, append func(o interface{}) (interface{}, error)) error {
return connection.ViewTx(func(tx portainer.Transaction) error {
return tx.GetAllWithKeyPrefix(bucketName, keyPrefix, obj, append)
return tx.GetAllWithKeyPrefix(bucketName, keyPrefix, obj, appendFn)
})
}
// BackupMetadata will return a copy of the boltdb sequence numbers for all buckets.
func (connection *DbConnection) BackupMetadata() (map[string]interface{}, error) {
buckets := map[string]interface{}{}
func (connection *DbConnection) BackupMetadata() (map[string]any, error) {
buckets := map[string]any{}
err := connection.View(func(tx *bolt.Tx) error {
err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
return tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
bucketName := string(name)
seqId := bucket.Sequence()
buckets[bucketName] = int(seqId)
return nil
})
return err
})
return buckets, err
}
// RestoreMetadata will restore the boltdb sequence numbers for all buckets.
func (connection *DbConnection) RestoreMetadata(s map[string]interface{}) error {
func (connection *DbConnection) RestoreMetadata(s map[string]any) error {
var err error
for bucketName, v := range s {
id, ok := v.(float64) // JSON ints are unmarshalled to interface as float64. See: https://pkg.go.dev/encoding/json#Decoder.Decode
if !ok {
log.Error().Str("bucket", bucketName).Msg("failed to restore metadata to bucket, skipped")
continue
}

View File

@@ -87,10 +87,7 @@ func Test_NeedsEncryptionMigration(t *testing.T) {
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
connection := DbConnection{Path: dir}
if tc.dbname == "both" {

View File

@@ -8,8 +8,8 @@ import (
bolt "go.etcd.io/bbolt"
)
func backupMetadata(connection *bolt.DB) (map[string]interface{}, error) {
buckets := map[string]interface{}{}
func backupMetadata(connection *bolt.DB) (map[string]any, error) {
buckets := map[string]any{}
err := connection.View(func(tx *bolt.Tx) error {
err := tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
@@ -39,7 +39,7 @@ func (c *DbConnection) ExportJSON(databasePath string, metadata bool) ([]byte, e
}
defer connection.Close()
backup := make(map[string]interface{})
backup := make(map[string]any)
if metadata {
meta, err := backupMetadata(connection)
if err != nil {
@@ -52,7 +52,7 @@ func (c *DbConnection) ExportJSON(databasePath string, metadata bool) ([]byte, e
err = connection.View(func(tx *bolt.Tx) error {
err = tx.ForEach(func(name []byte, bucket *bolt.Bucket) error {
bucketName := string(name)
var list []interface{}
var list []any
version := make(map[string]string)
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
@@ -60,7 +60,7 @@ func (c *DbConnection) ExportJSON(databasePath string, metadata bool) ([]byte, e
continue
}
var obj interface{}
var obj any
err := c.UnmarshalObject(v, &obj)
if err != nil {
log.Error().

View File

@@ -5,17 +5,16 @@ import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"fmt"
"io"
"github.com/pkg/errors"
"github.com/segmentio/encoding/json"
)
var errEncryptedStringTooShort = fmt.Errorf("encrypted string too short")
var errEncryptedStringTooShort = errors.New("encrypted string too short")
// MarshalObject encodes an object to binary format
func (connection *DbConnection) MarshalObject(object interface{}) ([]byte, error) {
func (connection *DbConnection) MarshalObject(object any) ([]byte, error) {
buf := &bytes.Buffer{}
// Special case for the VERSION bucket. Here we're not using json
@@ -39,7 +38,7 @@ func (connection *DbConnection) MarshalObject(object interface{}) ([]byte, error
}
// UnmarshalObject decodes an object from binary data
func (connection *DbConnection) UnmarshalObject(data []byte, object interface{}) error {
func (connection *DbConnection) UnmarshalObject(data []byte, object any) error {
var err error
if connection.getEncryptionKey() != nil {
data, err = decrypt(data, connection.getEncryptionKey())
@@ -47,8 +46,8 @@ func (connection *DbConnection) UnmarshalObject(data []byte, object interface{})
return errors.Wrap(err, "Failed decrypting object")
}
}
e := json.Unmarshal(data, object)
if e != nil {
if e := json.Unmarshal(data, object); e != nil {
// Special case for the VERSION bucket. Here we're not using json
// So we need to return it as a string
s, ok := object.(*string)
@@ -58,6 +57,7 @@ func (connection *DbConnection) UnmarshalObject(data []byte, object interface{})
*s = string(data)
}
return err
}
@@ -70,22 +70,20 @@ func encrypt(plaintext []byte, passphrase []byte) (encrypted []byte, err error)
if err != nil {
return encrypted, err
}
nonce := make([]byte, gcm.NonceSize())
if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return encrypted, err
}
ciphertextByte := gcm.Seal(
nonce,
nonce,
plaintext,
nil)
return ciphertextByte, nil
return gcm.Seal(nonce, nonce, plaintext, nil), nil
}
func decrypt(encrypted []byte, passphrase []byte) (plaintextByte []byte, err error) {
if string(encrypted) == "false" {
return []byte("false"), nil
}
block, err := aes.NewCipher(passphrase)
if err != nil {
return encrypted, errors.Wrap(err, "Error creating cypher block")
@@ -102,11 +100,8 @@ func decrypt(encrypted []byte, passphrase []byte) (plaintextByte []byte, err err
}
nonce, ciphertextByteClean := encrypted[:nonceSize], encrypted[nonceSize:]
plaintextByte, err = gcm.Open(
nil,
nonce,
ciphertextByteClean,
nil)
plaintextByte, err = gcm.Open(nil, nonce, ciphertextByteClean, nil)
if err != nil {
return encrypted, errors.Wrap(err, "Error decrypting text")
}

View File

@@ -25,7 +25,7 @@ func Test_MarshalObjectUnencrypted(t *testing.T) {
uuid := uuid.Must(uuid.NewV4())
tests := []struct {
object interface{}
object any
expected string
}{
{
@@ -57,7 +57,7 @@ func Test_MarshalObjectUnencrypted(t *testing.T) {
expected: uuid.String(),
},
{
object: map[string]interface{}{"key": "value"},
object: map[string]any{"key": "value"},
expected: `{"key":"value"}`,
},
{
@@ -73,11 +73,11 @@ func Test_MarshalObjectUnencrypted(t *testing.T) {
expected: `["1","2","3"]`,
},
{
object: []map[string]interface{}{{"key1": "value1"}, {"key2": "value2"}},
object: []map[string]any{{"key1": "value1"}, {"key2": "value2"}},
expected: `[{"key1":"value1"},{"key2":"value2"}]`,
},
{
object: []interface{}{1, "2", false, map[string]interface{}{"key1": "value1"}},
object: []any{1, "2", false, map[string]any{"key1": "value1"}},
expected: `[1,"2",false,{"key1":"value1"}]`,
},
}

View File

@@ -20,7 +20,7 @@ func (tx *DbTransaction) SetServiceName(bucketName string) error {
return err
}
func (tx *DbTransaction) GetObject(bucketName string, key []byte, object interface{}) error {
func (tx *DbTransaction) GetObject(bucketName string, key []byte, object any) error {
bucket := tx.tx.Bucket([]byte(bucketName))
value := bucket.Get(key)
@@ -31,7 +31,7 @@ func (tx *DbTransaction) GetObject(bucketName string, key []byte, object interfa
return tx.conn.UnmarshalObject(value, object)
}
func (tx *DbTransaction) UpdateObject(bucketName string, key []byte, object interface{}) error {
func (tx *DbTransaction) UpdateObject(bucketName string, key []byte, object any) error {
data, err := tx.conn.MarshalObject(object)
if err != nil {
return err
@@ -46,7 +46,7 @@ func (tx *DbTransaction) DeleteObject(bucketName string, key []byte) error {
return bucket.Delete(key)
}
func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, matchingFn func(o interface{}) (id int, ok bool)) error {
func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj any, matchingFn func(o any) (id int, ok bool)) error {
var ids []int
bucket := tx.tx.Bucket([]byte(bucketName))
@@ -74,16 +74,18 @@ func (tx *DbTransaction) DeleteAllObjects(bucketName string, obj interface{}, ma
func (tx *DbTransaction) GetNextIdentifier(bucketName string) int {
bucket := tx.tx.Bucket([]byte(bucketName))
id, err := bucket.NextSequence()
if err != nil {
log.Error().Err(err).Str("bucket", bucketName).Msg("failed to get the next identifer")
log.Error().Err(err).Str("bucket", bucketName).Msg("failed to get the next identifier")
return 0
}
return int(id)
}
func (tx *DbTransaction) CreateObject(bucketName string, fn func(uint64) (int, interface{})) error {
func (tx *DbTransaction) CreateObject(bucketName string, fn func(uint64) (int, any)) error {
bucket := tx.tx.Bucket([]byte(bucketName))
seqId, _ := bucket.NextSequence()
@@ -97,7 +99,7 @@ func (tx *DbTransaction) CreateObject(bucketName string, fn func(uint64) (int, i
return bucket.Put(tx.conn.ConvertToKey(id), data)
}
func (tx *DbTransaction) CreateObjectWithId(bucketName string, id int, obj interface{}) error {
func (tx *DbTransaction) CreateObjectWithId(bucketName string, id int, obj any) error {
bucket := tx.tx.Bucket([]byte(bucketName))
data, err := tx.conn.MarshalObject(obj)
if err != nil {
@@ -107,7 +109,7 @@ func (tx *DbTransaction) CreateObjectWithId(bucketName string, id int, obj inter
return bucket.Put(tx.conn.ConvertToKey(id), data)
}
func (tx *DbTransaction) CreateObjectWithStringId(bucketName string, id []byte, obj interface{}) error {
func (tx *DbTransaction) CreateObjectWithStringId(bucketName string, id []byte, obj any) error {
bucket := tx.tx.Bucket([]byte(bucketName))
data, err := tx.conn.MarshalObject(obj)
if err != nil {
@@ -117,7 +119,7 @@ func (tx *DbTransaction) CreateObjectWithStringId(bucketName string, id []byte,
return bucket.Put(id, data)
}
func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
func (tx *DbTransaction) GetAll(bucketName string, obj any, appendFn func(o any) (any, error)) error {
bucket := tx.tx.Bucket([]byte(bucketName))
return bucket.ForEach(func(k []byte, v []byte) error {
@@ -130,20 +132,7 @@ func (tx *DbTransaction) GetAll(bucketName string, obj interface{}, appendFn fun
})
}
func (tx *DbTransaction) GetAllWithJsoniter(bucketName string, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
bucket := tx.tx.Bucket([]byte(bucketName))
return bucket.ForEach(func(k []byte, v []byte) error {
err := tx.conn.UnmarshalObject(v, obj)
if err == nil {
obj, err = appendFn(obj)
}
return err
})
}
func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj interface{}, appendFn func(o interface{}) (interface{}, error)) error {
func (tx *DbTransaction) GetAllWithKeyPrefix(bucketName string, keyPrefix []byte, obj any, appendFn func(o any) (any, error)) error {
cursor := tx.tx.Bucket([]byte(bucketName)).Cursor()
for k, v := cursor.Seek(keyPrefix); k != nil && bytes.HasPrefix(k, keyPrefix); k, v = cursor.Next() {

View File

@@ -41,7 +41,7 @@ func (service *Service) GetAPIKeysByUserID(userID portainer.UserID) ([]portainer
err := service.Connection.GetAll(
BucketName,
&portainer.APIKey{},
func(obj interface{}) (interface{}, error) {
func(obj any) (any, error) {
record, ok := obj.(*portainer.APIKey)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
@@ -66,7 +66,7 @@ func (service *Service) GetAPIKeyByDigest(digest string) (*portainer.APIKey, err
err := service.Connection.GetAll(
BucketName,
&portainer.APIKey{},
func(obj interface{}) (interface{}, error) {
func(obj any) (any, error) {
key, ok := obj.(*portainer.APIKey)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to APIKey object")
@@ -95,7 +95,7 @@ func (service *Service) GetAPIKeyByDigest(digest string) (*portainer.APIKey, err
func (service *Service) Create(record *portainer.APIKey) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
record.ID = portainer.APIKeyID(id)
return int(record.ID), record

View File

@@ -31,7 +31,7 @@ func (service BaseDataServiceTx[T, I]) Read(ID I) (*T, error) {
func (service BaseDataServiceTx[T, I]) ReadAll() ([]T, error) {
var collection = make([]T, 0)
return collection, service.Tx.GetAllWithJsoniter(
return collection, service.Tx.GetAll(
service.Bucket,
new(T),
AppendFn(&collection),

View File

@@ -19,7 +19,7 @@ func (service ServiceTx) UpdateEdgeGroupFunc(ID portainer.EdgeGroupID, updateFun
func (service ServiceTx) Create(group *portainer.EdgeGroup) error {
return service.Tx.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
group.ID = portainer.EdgeGroupID(id)
return int(group.ID), group
},

View File

@@ -24,7 +24,7 @@ func (service ServiceTx) EdgeStacks() ([]portainer.EdgeStack, error) {
err := service.tx.GetAll(
BucketName,
&portainer.EdgeStack{},
func(obj interface{}) (interface{}, error) {
func(obj any) (any, error) {
stack, ok := obj.(*portainer.EdgeStack)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to EdgeStack object")

View File

@@ -6,6 +6,8 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/rs/zerolog/log"
)
// BucketName represents the name of the bucket where this service stores data.
@@ -157,6 +159,7 @@ func (service *Service) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.
return true
}
}
return false
}),
)
@@ -166,11 +169,13 @@ func (service *Service) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer.
func (service *Service) GetNextIdentifier() int {
var identifier int
service.connection.UpdateTx(func(tx portainer.Transaction) error {
if err := service.connection.UpdateTx(func(tx portainer.Transaction) error {
identifier = service.Tx(tx).GetNextIdentifier()
return nil
})
}); err != nil {
log.Error().Err(err).Str("bucket", BucketName).Msg("could not get the next identifier")
}
return identifier
}

View File

@@ -20,10 +20,10 @@ func (service ServiceTx) BucketName() string {
// Endpoint returns an environment(endpoint) by ID.
func (service ServiceTx) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint, error) {
var endpoint portainer.Endpoint
identifier := service.service.connection.ConvertToKey(int(ID))
err := service.tx.GetObject(BucketName, identifier, &endpoint)
if err != nil {
if err := service.tx.GetObject(BucketName, identifier, &endpoint); err != nil {
return nil, err
}
@@ -36,8 +36,7 @@ func (service ServiceTx) Endpoint(ID portainer.EndpointID) (*portainer.Endpoint,
func (service ServiceTx) UpdateEndpoint(ID portainer.EndpointID, endpoint *portainer.Endpoint) error {
identifier := service.service.connection.ConvertToKey(int(ID))
err := service.tx.UpdateObject(BucketName, identifier, endpoint)
if err != nil {
if err := service.tx.UpdateObject(BucketName, identifier, endpoint); err != nil {
return err
}
@@ -45,6 +44,7 @@ func (service ServiceTx) UpdateEndpoint(ID portainer.EndpointID, endpoint *porta
if len(endpoint.EdgeID) > 0 {
service.service.idxEdgeID[endpoint.EdgeID] = ID
}
service.service.heartbeats.Store(ID, endpoint.LastCheckInDate)
service.service.mu.Unlock()
@@ -57,8 +57,7 @@ func (service ServiceTx) UpdateEndpoint(ID portainer.EndpointID, endpoint *porta
func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
identifier := service.service.connection.ConvertToKey(int(ID))
err := service.tx.DeleteObject(BucketName, identifier)
if err != nil {
if err := service.tx.DeleteObject(BucketName, identifier); err != nil {
return err
}
@@ -70,6 +69,7 @@ func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
break
}
}
service.service.heartbeats.Delete(ID)
service.service.mu.Unlock()
@@ -82,7 +82,7 @@ func (service ServiceTx) DeleteEndpoint(ID portainer.EndpointID) error {
func (service ServiceTx) Endpoints() ([]portainer.Endpoint, error) {
var endpoints = make([]portainer.Endpoint, 0)
return endpoints, service.tx.GetAllWithJsoniter(
return endpoints, service.tx.GetAll(
BucketName,
&portainer.Endpoint{},
dataservices.AppendFn(&endpoints),
@@ -107,8 +107,7 @@ func (service ServiceTx) UpdateHeartbeat(endpointID portainer.EndpointID) {
// CreateEndpoint assign an ID to a new environment(endpoint) and saves it.
func (service ServiceTx) Create(endpoint *portainer.Endpoint) error {
err := service.tx.CreateObjectWithId(BucketName, int(endpoint.ID), endpoint)
if err != nil {
if err := service.tx.CreateObjectWithId(BucketName, int(endpoint.ID), endpoint); err != nil {
return err
}
@@ -116,6 +115,7 @@ func (service ServiceTx) Create(endpoint *portainer.Endpoint) error {
if len(endpoint.EdgeID) > 0 {
service.service.idxEdgeID[endpoint.EdgeID] = endpoint.ID
}
service.service.heartbeats.Store(endpoint.ID, endpoint.LastCheckInDate)
service.service.mu.Unlock()
@@ -134,6 +134,7 @@ func (service ServiceTx) EndpointsByTeamID(teamID portainer.TeamID) ([]portainer
return true
}
}
return false
}),
)

View File

@@ -41,7 +41,7 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
func (service *Service) Create(endpointGroup *portainer.EndpointGroup) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
endpointGroup.ID = portainer.EndpointGroupID(id)
return int(endpointGroup.ID), endpointGroup
},

View File

@@ -13,7 +13,7 @@ type ServiceTx struct {
func (service ServiceTx) Create(endpointGroup *portainer.EndpointGroup) error {
return service.Tx.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
endpointGroup.ID = portainer.EndpointGroupID(id)
return int(endpointGroup.ID), endpointGroup
},

View File

@@ -32,8 +32,7 @@ func (service *Service) RegisterUpdateStackFunction(
// NewService creates a new instance of a service.
func NewService(connection portainer.Connection) (*Service, error) {
err := connection.SetServiceName(BucketName)
if err != nil {
if err := connection.SetServiceName(BucketName); err != nil {
return nil, err
}
@@ -65,8 +64,7 @@ func (service *Service) EndpointRelation(endpointID portainer.EndpointID) (*port
var endpointRelation portainer.EndpointRelation
identifier := service.connection.ConvertToKey(int(endpointID))
err := service.connection.GetObject(BucketName, identifier, &endpointRelation)
if err != nil {
if err := service.connection.GetObject(BucketName, identifier, &endpointRelation); err != nil {
return nil, err
}
@@ -161,19 +159,24 @@ func (service *Service) updateEdgeStacksAfterRelationChange(previousRelationStat
// list how many time this stack is referenced in all relations
// in order to update the stack deployments count
for refStackId, refStackEnabled := range stacksToUpdate {
if refStackEnabled {
numDeployments := 0
for _, r := range relations {
for sId, enabled := range r.EdgeStacks {
if enabled && sId == refStackId {
numDeployments += 1
}
if !refStackEnabled {
continue
}
numDeployments := 0
for _, r := range relations {
for sId, enabled := range r.EdgeStacks {
if enabled && sId == refStackId {
numDeployments += 1
}
}
}
service.updateStackFn(refStackId, func(edgeStack *portainer.EdgeStack) {
edgeStack.NumDeployments = numDeployments
})
if err := service.updateStackFn(refStackId, func(edgeStack *portainer.EdgeStack) {
edgeStack.NumDeployments = numDeployments
}); err != nil {
log.Error().Err(err).Msg("could not update the number of deployments")
}
}
}

View File

@@ -33,8 +33,7 @@ func (service ServiceTx) EndpointRelation(endpointID portainer.EndpointID) (*por
var endpointRelation portainer.EndpointRelation
identifier := service.service.connection.ConvertToKey(int(endpointID))
err := service.tx.GetObject(BucketName, identifier, &endpointRelation)
if err != nil {
if err := service.tx.GetObject(BucketName, identifier, &endpointRelation); err != nil {
return nil, err
}
@@ -129,19 +128,23 @@ func (service ServiceTx) updateEdgeStacksAfterRelationChange(previousRelationSta
// list how many time this stack is referenced in all relations
// in order to update the stack deployments count
for refStackId, refStackEnabled := range stacksToUpdate {
if refStackEnabled {
numDeployments := 0
for _, r := range relations {
for sId, enabled := range r.EdgeStacks {
if enabled && sId == refStackId {
numDeployments += 1
}
if !refStackEnabled {
continue
}
numDeployments := 0
for _, r := range relations {
for sId, enabled := range r.EdgeStacks {
if enabled && sId == refStackId {
numDeployments += 1
}
}
}
service.service.updateStackFnTx(service.tx, refStackId, func(edgeStack *portainer.EdgeStack) {
edgeStack.NumDeployments = numDeployments
})
if err := service.service.updateStackFnTx(service.tx, refStackId, func(edgeStack *portainer.EdgeStack) {
edgeStack.NumDeployments = numDeployments
}); err != nil {
log.Error().Err(err).Msg("could not update the number of deployments")
}
}
}

View File

@@ -1,43 +0,0 @@
package fdoprofile
import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
)
// BucketName represents the name of the bucket where this service stores data.
const BucketName = "fdo_profiles"
// Service represents a service for managingFDO Profiles data.
type Service struct {
dataservices.BaseDataService[portainer.FDOProfile, portainer.FDOProfileID]
}
// NewService creates a new instance of a service.
func NewService(connection portainer.Connection) (*Service, error) {
err := connection.SetServiceName(BucketName)
if err != nil {
return nil, err
}
return &Service{
BaseDataService: dataservices.BaseDataService[portainer.FDOProfile, portainer.FDOProfileID]{
Bucket: BucketName,
Connection: connection,
},
}, nil
}
// Create assign an ID to a new FDO Profile and saves it.
func (service *Service) Create(FDOProfile *portainer.FDOProfile) error {
return service.Connection.CreateObjectWithId(
BucketName,
int(FDOProfile.ID),
FDOProfile,
)
}
// GetNextIdentifier returns the next identifier for a FDO Profile.
func (service *Service) GetNextIdentifier() int {
return service.Connection.GetNextIdentifier(BucketName)
}

View File

@@ -45,7 +45,7 @@ func (service *Service) HelmUserRepositoryByUserID(userID portainer.UserID) ([]p
func (service *Service) Create(record *portainer.HelmUserRepository) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
record.ID = portainer.HelmUserRepositoryID(id)
return int(record.ID), record
},

View File

@@ -17,8 +17,8 @@ func IsErrObjectNotFound(e error) bool {
}
// AppendFn appends elements to the given collection slice
func AppendFn[T any](collection *[]T) func(obj interface{}) (interface{}, error) {
return func(obj interface{}) (interface{}, error) {
func AppendFn[T any](collection *[]T) func(obj any) (any, error) {
return func(obj any) (any, error) {
element, ok := obj.(*T)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
@@ -32,8 +32,8 @@ func AppendFn[T any](collection *[]T) func(obj interface{}) (interface{}, error)
}
// FilterFn appends elements to the given collection when the predicate is true
func FilterFn[T any](collection *[]T, predicate func(T) bool) func(obj interface{}) (interface{}, error) {
return func(obj interface{}) (interface{}, error) {
func FilterFn[T any](collection *[]T, predicate func(T) bool) func(obj any) (any, error) {
return func(obj any) (any, error) {
element, ok := obj.(*T)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")
@@ -50,8 +50,8 @@ func FilterFn[T any](collection *[]T, predicate func(T) bool) func(obj interface
// FirstFn sets the element to the first one that satisfies the predicate and stops the computation, returns ErrStop on
// success
func FirstFn[T any](element *T, predicate func(T) bool) func(obj interface{}) (interface{}, error) {
return func(obj interface{}) (interface{}, error) {
func FirstFn[T any](element *T, predicate func(T) bool) func(obj any) (any, error) {
return func(obj any) (any, error) {
e, ok := obj.(*T)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("type assertion failed")

View File

@@ -15,7 +15,6 @@ type (
Endpoint() EndpointService
EndpointGroup() EndpointGroupService
EndpointRelation() EndpointRelationService
FDOProfile() FDOProfileService
HelmUserRepository() HelmUserRepositoryService
Registry() RegistryService
ResourceControl() ResourceControlService
@@ -120,12 +119,6 @@ type (
BucketName() string
}
// FDOProfileService represents a service to manage FDO Profiles
FDOProfileService interface {
BaseCRUD[portainer.FDOProfile, portainer.FDOProfileID]
GetNextIdentifier() int
}
// HelmUserRepositoryService represents a service to manage HelmUserRepositories
HelmUserRepositoryService interface {
BaseCRUD[portainer.HelmUserRepository, portainer.HelmUserRepositoryID]

View File

@@ -64,7 +64,7 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
}
func (s ServiceTx) Create(config *portainer.PendingAction) error {
return s.Tx.CreateObject(BucketName, func(id uint64) (int, interface{}) {
return s.Tx.CreateObject(BucketName, func(id uint64) (int, any) {
config.ID = portainer.PendingActionID(id)
config.CreatedAt = time.Now().Unix()

View File

@@ -42,7 +42,7 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
func (service *Service) Create(registry *portainer.Registry) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
registry.ID = portainer.RegistryID(id)
return int(registry.ID), registry
},

View File

@@ -13,7 +13,7 @@ type ServiceTx struct {
func (service ServiceTx) Create(registry *portainer.Registry) error {
return service.Tx.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
registry.ID = portainer.RegistryID(id)
return int(registry.ID), registry
},

View File

@@ -52,7 +52,7 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
err := service.Connection.GetAll(
BucketName,
&portainer.ResourceControl{},
func(obj interface{}) (interface{}, error) {
func(obj any) (any, error) {
rc, ok := obj.(*portainer.ResourceControl)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
@@ -84,7 +84,7 @@ func (service *Service) ResourceControlByResourceIDAndType(resourceID string, re
func (service *Service) Create(resourceControl *portainer.ResourceControl) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
resourceControl.ID = portainer.ResourceControlID(id)
return int(resourceControl.ID), resourceControl
},

View File

@@ -23,7 +23,7 @@ func (service ServiceTx) ResourceControlByResourceIDAndType(resourceID string, r
err := service.Tx.GetAll(
BucketName,
&portainer.ResourceControl{},
func(obj interface{}) (interface{}, error) {
func(obj any) (any, error) {
rc, ok := obj.(*portainer.ResourceControl)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to ResourceControl object")
@@ -55,7 +55,7 @@ func (service ServiceTx) ResourceControlByResourceIDAndType(resourceID string, r
func (service ServiceTx) Create(resourceControl *portainer.ResourceControl) error {
return service.Tx.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
resourceControl.ID = portainer.ResourceControlID(id)
return int(resourceControl.ID), resourceControl
},

View File

@@ -42,7 +42,7 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
func (service *Service) Create(role *portainer.Role) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
role.ID = portainer.RoleID(id)
return int(role.ID), role
},

View File

@@ -13,7 +13,7 @@ type ServiceTx struct {
func (service ServiceTx) Create(role *portainer.Role) error {
return service.Tx.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
role.ID = portainer.RoleID(id)
return int(role.ID), role
},

View File

@@ -33,7 +33,7 @@ func TestService_StackByWebhookID(t *testing.T) {
b := stackBuilder{t: t, store: store}
b.createNewStack(newGuidString(t))
for i := 0; i < 10; i++ {
for range 10 {
b.createNewStack("")
}
webhookID := newGuidString(t)

View File

@@ -42,7 +42,7 @@ func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
func (service *Service) Create(tag *portainer.Tag) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
tag.ID = portainer.TagID(id)
return int(tag.ID), tag
},

View File

@@ -15,7 +15,7 @@ type ServiceTx struct {
func (service ServiceTx) Create(tag *portainer.Tag) error {
return service.Tx.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
tag.ID = portainer.TagID(id)
return int(tag.ID), tag
},

View File

@@ -19,8 +19,7 @@ type Service struct {
// NewService creates a new instance of a service.
func NewService(connection portainer.Connection) (*Service, error) {
err := connection.SetServiceName(BucketName)
if err != nil {
if err := connection.SetServiceName(BucketName); err != nil {
return nil, err
}
@@ -32,6 +31,16 @@ func NewService(connection portainer.Connection) (*Service, error) {
}, nil
}
func (service *Service) Tx(tx portainer.Transaction) ServiceTx {
return ServiceTx{
BaseDataServiceTx: dataservices.BaseDataServiceTx[portainer.Team, portainer.TeamID]{
Bucket: BucketName,
Connection: service.Connection,
Tx: tx,
},
}
}
// TeamByName returns a team by name.
func (service *Service) TeamByName(name string) (*portainer.Team, error) {
var t portainer.Team
@@ -59,7 +68,7 @@ func (service *Service) TeamByName(name string) (*portainer.Team, error) {
func (service *Service) Create(team *portainer.Team) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
team.ID = portainer.TeamID(id)
return int(team.ID), team
},

View File

@@ -0,0 +1,48 @@
package team
import (
"errors"
"strings"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dserrors "github.com/portainer/portainer/api/dataservices/errors"
)
type ServiceTx struct {
dataservices.BaseDataServiceTx[portainer.Team, portainer.TeamID]
}
// TeamByName returns a team by name.
func (service ServiceTx) TeamByName(name string) (*portainer.Team, error) {
var t portainer.Team
err := service.Tx.GetAll(
BucketName,
&portainer.Team{},
dataservices.FirstFn(&t, func(e portainer.Team) bool {
return strings.EqualFold(e.Name, name)
}),
)
if errors.Is(err, dataservices.ErrStop) {
return &t, nil
}
if err == nil {
return nil, dserrors.ErrObjectNotFound
}
return nil, err
}
// CreateTeam creates a new Team.
func (service ServiceTx) Create(team *portainer.Team) error {
return service.Tx.CreateObject(
BucketName,
func(id uint64) (int, any) {
team.ID = portainer.TeamID(id)
return int(team.ID), team
},
)
}

View File

@@ -72,7 +72,7 @@ func (service *Service) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]port
func (service *Service) Create(membership *portainer.TeamMembership) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
membership.ID = portainer.TeamMembershipID(id)
return int(membership.ID), membership
},
@@ -84,8 +84,8 @@ func (service *Service) DeleteTeamMembershipByUserID(userID portainer.UserID) er
return service.Connection.DeleteAllObjects(
BucketName,
&portainer.TeamMembership{},
func(obj interface{}) (id int, ok bool) {
membership, ok := obj.(portainer.TeamMembership)
func(obj any) (id int, ok bool) {
membership, ok := obj.(*portainer.TeamMembership)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
@@ -105,8 +105,8 @@ func (service *Service) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) er
return service.Connection.DeleteAllObjects(
BucketName,
&portainer.TeamMembership{},
func(obj interface{}) (id int, ok bool) {
membership, ok := obj.(portainer.TeamMembership)
func(obj any) (id int, ok bool) {
membership, ok := obj.(*portainer.TeamMembership)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)
@@ -125,8 +125,8 @@ func (service *Service) DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.T
return service.Connection.DeleteAllObjects(
BucketName,
&portainer.TeamMembership{},
func(obj interface{}) (id int, ok bool) {
membership, ok := obj.(portainer.TeamMembership)
func(obj any) (id int, ok bool) {
membership, ok := obj.(*portainer.TeamMembership)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
//return fmt.Errorf("Failed to convert to TeamMembership object: %s", obj)

View File

@@ -43,7 +43,7 @@ func (service ServiceTx) TeamMembershipsByTeamID(teamID portainer.TeamID) ([]por
func (service ServiceTx) Create(membership *portainer.TeamMembership) error {
return service.Tx.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
membership.ID = portainer.TeamMembershipID(id)
return int(membership.ID), membership
},
@@ -55,7 +55,7 @@ func (service ServiceTx) DeleteTeamMembershipByUserID(userID portainer.UserID) e
return service.Tx.DeleteAllObjects(
BucketName,
&portainer.TeamMembership{},
func(obj interface{}) (id int, ok bool) {
func(obj any) (id int, ok bool) {
membership, ok := obj.(portainer.TeamMembership)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
@@ -76,7 +76,7 @@ func (service ServiceTx) DeleteTeamMembershipByTeamID(teamID portainer.TeamID) e
return service.Tx.DeleteAllObjects(
BucketName,
&portainer.TeamMembership{},
func(obj interface{}) (id int, ok bool) {
func(obj any) (id int, ok bool) {
membership, ok := obj.(portainer.TeamMembership)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")
@@ -96,7 +96,7 @@ func (service ServiceTx) DeleteTeamMembershipByTeamIDAndUserID(teamID portainer.
return service.Tx.DeleteAllObjects(
BucketName,
&portainer.TeamMembership{},
func(obj interface{}) (id int, ok bool) {
func(obj any) (id int, ok bool) {
membership, ok := obj.(portainer.TeamMembership)
if !ok {
log.Debug().Str("obj", fmt.Sprintf("%#v", obj)).Msg("failed to convert to TeamMembership object")

View File

@@ -53,7 +53,7 @@ func (service ServiceTx) UsersByRole(role portainer.UserRole) ([]portainer.User,
func (service ServiceTx) Create(user *portainer.User) error {
return service.Tx.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
user.ID = portainer.UserID(id)
user.Username = strings.ToLower(user.Username)

View File

@@ -82,7 +82,7 @@ func (service *Service) UsersByRole(role portainer.UserRole) ([]portainer.User,
func (service *Service) Create(user *portainer.User) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
user.ID = portainer.UserID(id)
user.Username = strings.ToLower(user.Username)

View File

@@ -81,7 +81,7 @@ func (service *Service) WebhookByToken(token string) (*portainer.Webhook, error)
func (service *Service) Create(webhook *portainer.Webhook) error {
return service.Connection.CreateObject(
BucketName,
func(id uint64) (int, interface{}) {
func(id uint64) (int, any) {
webhook.ID = portainer.WebhookID(id)
return int(webhook.ID), webhook
},

View File

@@ -9,7 +9,7 @@ import (
"path/filepath"
"testing"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/database/boltdb"
"github.com/portainer/portainer/api/database/models"
@@ -321,7 +321,7 @@ func migrateDBTestHelper(t *testing.T, srcPath, wantPath string, overrideInstanc
// importJSON reads input JSON and commits it to a portainer datastore.Store.
// Errors are logged with the testing package.
func importJSON(t *testing.T, r io.Reader, store *Store) error {
objects := make(map[string]interface{})
objects := make(map[string]any)
// Parse json into map of objects.
d := json.NewDecoder(r)
@@ -337,9 +337,9 @@ func importJSON(t *testing.T, r io.Reader, store *Store) error {
for k, v := range objects {
switch k {
case "version":
versions, ok := v.(map[string]interface{})
versions, ok := v.(map[string]any)
if !ok {
t.Logf("failed casting %s to map[string]interface{}", k)
t.Logf("failed casting %s to map[string]any", k)
}
// New format db
@@ -404,9 +404,9 @@ func importJSON(t *testing.T, r io.Reader, store *Store) error {
}
case "dockerhub":
obj, ok := v.([]interface{})
obj, ok := v.([]any)
if !ok {
t.Logf("failed to cast %s to []interface{}", k)
t.Logf("failed to cast %s to []any", k)
}
err := con.CreateObjectWithStringId(
k,
@@ -418,9 +418,9 @@ func importJSON(t *testing.T, r io.Reader, store *Store) error {
}
case "ssl":
obj, ok := v.(map[string]interface{})
obj, ok := v.(map[string]any)
if !ok {
t.Logf("failed to case %s to map[string]interface{}", k)
t.Logf("failed to case %s to map[string]any", k)
}
err := con.CreateObjectWithStringId(
k,
@@ -432,9 +432,9 @@ func importJSON(t *testing.T, r io.Reader, store *Store) error {
}
case "settings":
obj, ok := v.(map[string]interface{})
obj, ok := v.(map[string]any)
if !ok {
t.Logf("failed to case %s to map[string]interface{}", k)
t.Logf("failed to case %s to map[string]any", k)
}
err := con.CreateObjectWithStringId(
k,
@@ -446,9 +446,9 @@ func importJSON(t *testing.T, r io.Reader, store *Store) error {
}
case "tunnel_server":
obj, ok := v.(map[string]interface{})
obj, ok := v.(map[string]any)
if !ok {
t.Logf("failed to case %s to map[string]interface{}", k)
t.Logf("failed to case %s to map[string]any", k)
}
err := con.CreateObjectWithStringId(
k,
@@ -462,18 +462,18 @@ func importJSON(t *testing.T, r io.Reader, store *Store) error {
continue
default:
objlist, ok := v.([]interface{})
objlist, ok := v.([]any)
if !ok {
t.Logf("failed to cast %s to []interface{}", k)
t.Logf("failed to cast %s to []any", k)
}
for _, obj := range objlist {
value, ok := obj.(map[string]interface{})
value, ok := obj.(map[string]any)
if !ok {
t.Logf("failed to cast %v to map[string]interface{}", obj)
t.Logf("failed to cast %v to map[string]any", obj)
} else {
var ok bool
var id interface{}
var id any
switch k {
case "endpoint_relations":
// TODO: need to make into an int, then do that weird

View File

@@ -12,13 +12,13 @@ const dummyLogoURL = "example.com"
// initTestingDBConn creates a settings service with raw database DB connection
// for unit testing usage only since using NewStore will cause cycle import inside migrator pkg
func initTestingSettingsService(dbConn portainer.Connection, preSetObj map[string]interface{}) error {
func initTestingSettingsService(dbConn portainer.Connection, preSetObj map[string]any) error {
//insert a obj
return dbConn.UpdateObject("settings", []byte("SETTINGS"), preSetObj)
}
func setup(store *Store) error {
dummySettingsObj := map[string]interface{}{
dummySettingsObj := map[string]any{
"LogoURL": dummyLogoURL,
}

View File

@@ -111,5 +111,6 @@ func (store *Store) finishMigrateLegacyVersion(versionToWrite *models.Version) e
store.connection.DeleteObject(bucketName, []byte(legacyDBVersionKey))
store.connection.DeleteObject(bucketName, []byte(legacyEditionKey))
store.connection.DeleteObject(bucketName, []byte(legacyInstanceKey))
return err
}

View File

@@ -7,7 +7,7 @@ import (
"github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
"github.com/rs/zerolog/log"
)
@@ -15,7 +15,7 @@ func migrationError(err error, context string) error {
return errors.Wrap(err, "failed in "+context)
}
func GetFunctionName(i interface{}) string {
func GetFunctionName(i any) string {
return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
}
@@ -39,20 +39,19 @@ func (m *Migrator) Migrate() error {
latestMigrations := m.LatestMigrations()
if latestMigrations.Version.Equal(schemaVersion) &&
version.MigratorCount != len(latestMigrations.MigrationFuncs) {
err := runMigrations(latestMigrations.MigrationFuncs)
if err != nil {
if err := runMigrations(latestMigrations.MigrationFuncs); err != nil {
return err
}
newMigratorCount = len(latestMigrations.MigrationFuncs)
}
} else {
// regular path when major/minor/patch versions differ
for _, migration := range m.migrations {
if schemaVersion.LessThan(migration.Version) {
log.Info().Msgf("migrating data to %s", migration.Version.String())
err := runMigrations(migration.MigrationFuncs)
if err != nil {
if err := runMigrations(migration.MigrationFuncs); err != nil {
return err
}
}
@@ -63,16 +62,14 @@ func (m *Migrator) Migrate() error {
}
}
err = m.Always()
if err != nil {
if err := m.Always(); err != nil {
return migrationError(err, "Always migrations returned error")
}
version.SchemaVersion = portainer.APIVersion
version.MigratorCount = newMigratorCount
err = m.versionService.UpdateVersion(version)
if err != nil {
if err := m.versionService.UpdateVersion(version); err != nil {
return migrationError(err, "StoreDBVersion")
}
@@ -99,6 +96,7 @@ func (m *Migrator) NeedsMigration() bool {
// In this particular instance we should log a fatal error
if m.CurrentDBEdition() != portainer.PortainerCE {
log.Fatal().Msg("the Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
return false
}

View File

@@ -7,6 +7,7 @@ import (
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/chisel/crypto"
"github.com/portainer/portainer/api/dataservices"
"github.com/rs/zerolog/log"
)
@@ -37,9 +38,11 @@ func (m *Migrator) convertSeedToPrivateKeyForDB100() error {
log.Info().Msg("ServerInfo object not found")
return nil
}
log.Error().
Err(err).
Msg("Failed to read ServerInfo from DB")
return err
}
@@ -49,14 +52,15 @@ func (m *Migrator) convertSeedToPrivateKeyForDB100() error {
log.Error().
Err(err).
Msg("Failed to read ServerInfo from DB")
return err
}
err = m.fileService.StoreChiselPrivateKey(key)
if err != nil {
if err := m.fileService.StoreChiselPrivateKey(key); err != nil {
log.Error().
Err(err).
Msg("Failed to save Chisel private key to disk")
return err
}
} else {
@@ -64,14 +68,14 @@ func (m *Migrator) convertSeedToPrivateKeyForDB100() error {
}
serverInfo.PrivateKeySeed = ""
err = m.TunnelServerService.UpdateInfo(serverInfo)
if err != nil {
if err := m.TunnelServerService.UpdateInfo(serverInfo); err != nil {
log.Error().
Err(err).
Msg("Failed to clean private key seed in DB")
} else {
log.Info().Msg("Success to migrate private key seed to private key file")
}
return err
}
@@ -84,9 +88,8 @@ func (m *Migrator) updateEdgeStackStatusForDB100() error {
}
for _, edgeStack := range edgeStacks {
for environmentID, environmentStatus := range edgeStack.Status {
// skip if status is already updated
// Skip if status is already updated
if len(environmentStatus.Status) > 0 {
continue
}
@@ -146,8 +149,7 @@ func (m *Migrator) updateEdgeStackStatusForDB100() error {
edgeStack.Status[environmentID] = environmentStatus
}
err = m.edgeStackService.UpdateEdgeStack(edgeStack.ID, &edgeStack)
if err != nil {
if err := m.edgeStackService.UpdateEdgeStack(edgeStack.ID, &edgeStack); err != nil {
return err
}
}

View File

@@ -32,8 +32,8 @@ func (m *Migrator) updateStacksToDB24() error {
for idx := range stacks {
stack := &stacks[idx]
stack.Status = portainer.StackStatusActive
err := m.stackService.Update(stack.ID, stack)
if err != nil {
if err := m.stackService.Update(stack.ID, stack); err != nil {
return err
}
}

View File

@@ -3,7 +3,7 @@ package migrator
import (
"errors"
"github.com/Masterminds/semver"
"github.com/Masterminds/semver/v3"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/database/models"
"github.com/portainer/portainer/api/dataservices/dockerhub"
@@ -13,7 +13,6 @@ import (
"github.com/portainer/portainer/api/dataservices/endpointgroup"
"github.com/portainer/portainer/api/dataservices/endpointrelation"
"github.com/portainer/portainer/api/dataservices/extension"
"github.com/portainer/portainer/api/dataservices/fdoprofile"
"github.com/portainer/portainer/api/dataservices/pendingactions"
"github.com/portainer/portainer/api/dataservices/registry"
"github.com/portainer/portainer/api/dataservices/resourcecontrol"
@@ -41,7 +40,6 @@ type (
endpointService *endpoint.Service
endpointRelationService *endpointrelation.Service
extensionService *extension.Service
fdoProfilesService *fdoprofile.Service
registryService *registry.Service
resourceControlService *resourcecontrol.Service
roleService *role.Service
@@ -69,7 +67,6 @@ type (
EndpointService *endpoint.Service
EndpointRelationService *endpointrelation.Service
ExtensionService *extension.Service
FDOProfilesService *fdoprofile.Service
RegistryService *registry.Service
ResourceControlService *resourcecontrol.Service
RoleService *role.Service
@@ -99,7 +96,6 @@ func NewMigrator(parameters *MigratorParameters) *Migrator {
endpointService: parameters.EndpointService,
endpointRelationService: parameters.EndpointRelationService,
extensionService: parameters.ExtensionService,
fdoProfilesService: parameters.FDOProfilesService,
registryService: parameters.RegistryService,
resourceControlService: parameters.ResourceControlService,
roleService: parameters.RoleService,

View File

@@ -90,7 +90,7 @@ func (migrator *PostInitMigrator) MigrateEnvironment(environment *portainer.Endp
switch {
case endpointutils.IsKubernetesEndpoint(environment):
// get the kubeclient for the environment, and skip all kube migrations if there's an error
kubeclient, err := migrator.kubeFactory.GetKubeClient(environment)
kubeclient, err := migrator.kubeFactory.GetPrivilegedKubeClient(environment)
if err != nil {
log.Error().Err(err).Msgf("Error creating kubeclient for environment: %d", environment.ID)
return err

View File

@@ -17,7 +17,6 @@ import (
"github.com/portainer/portainer/api/dataservices/endpointgroup"
"github.com/portainer/portainer/api/dataservices/endpointrelation"
"github.com/portainer/portainer/api/dataservices/extension"
"github.com/portainer/portainer/api/dataservices/fdoprofile"
"github.com/portainer/portainer/api/dataservices/helmuserrepository"
"github.com/portainer/portainer/api/dataservices/pendingactions"
"github.com/portainer/portainer/api/dataservices/registry"
@@ -55,7 +54,6 @@ type Store struct {
EndpointService *endpoint.Service
EndpointRelationService *endpointrelation.Service
ExtensionService *extension.Service
FDOProfilesService *fdoprofile.Service
HelmUserRepositoryService *helmuserrepository.Service
RegistryService *registry.Service
ResourceControlService *resourcecontrol.Service
@@ -138,12 +136,6 @@ func (store *Store) initServices() error {
}
store.ExtensionService = extensionService
fdoProfilesService, err := fdoprofile.NewService(store.connection)
if err != nil {
return err
}
store.FDOProfilesService = fdoProfilesService
helmUserRepositoryService, err := helmuserrepository.NewService(store.connection)
if err != nil {
return err
@@ -289,11 +281,6 @@ func (store *Store) EndpointRelation() dataservices.EndpointRelationService {
return store.EndpointRelationService
}
// FDOProfile gives access to the FDOProfile data management layer
func (store *Store) FDOProfile() dataservices.FDOProfileService {
return store.FDOProfilesService
}
// HelmUserRepository access the helm user repository settings
func (store *Store) HelmUserRepository() dataservices.HelmUserRepositoryService {
return store.HelmUserRepositoryService
@@ -398,11 +385,10 @@ type storeExport struct {
User []portainer.User `json:"users,omitempty"`
Version models.Version `json:"version,omitempty"`
Webhook []portainer.Webhook `json:"webhooks,omitempty"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
}
func (store *Store) Export(filename string) (err error) {
backup := storeExport{}
if c, err := store.CustomTemplate().ReadAll(); err != nil {
@@ -606,6 +592,7 @@ func (store *Store) Export(filename string) (err error) {
if err != nil {
return err
}
return os.WriteFile(filename, b, 0600)
}

View File

@@ -44,7 +44,6 @@ func (tx *StoreTx) EndpointRelation() dataservices.EndpointRelationService {
return tx.store.EndpointRelationService.Tx(tx.tx)
}
func (tx *StoreTx) FDOProfile() dataservices.FDOProfileService { return nil }
func (tx *StoreTx) HelmUserRepository() dataservices.HelmUserRepositoryService { return nil }
func (tx *StoreTx) Registry() dataservices.RegistryService {
@@ -83,7 +82,9 @@ func (tx *StoreTx) TeamMembership() dataservices.TeamMembershipService {
return tx.store.TeamMembershipService.Tx(tx.tx)
}
func (tx *StoreTx) Team() dataservices.TeamService { return nil }
func (tx *StoreTx) Team() dataservices.TeamService {
return tx.store.TeamService.Tx(tx.tx)
}
func (tx *StoreTx) TunnelServer() dataservices.TunnelServerService { return nil }

View File

@@ -38,6 +38,7 @@
"TenantID": ""
},
"ComposeSyntaxMaxVersion": "",
"ContainerEngine": "",
"Edge": {
"AsyncMode": false,
"CommandInterval": 0,
@@ -583,7 +584,6 @@
"AuthenticationMethod": 1,
"BlackListedLabels": [],
"Edge": {
"AsyncMode": false,
"CommandInterval": 0,
"PingInterval": 0,
"SnapshotInterval": 0
@@ -602,7 +602,7 @@
"RequiredPasswordLength": 12
},
"KubeconfigExpiry": "0",
"KubectlShellImage": "portainer/kubectl-shell",
"KubectlShellImage": "portainer/kubectl-shell:2.22.0",
"LDAPSettings": {
"AnonymousMode": true,
"AutoCreateUsers": true,
@@ -644,17 +644,10 @@
"Scopes": "",
"UserIdentifier": ""
},
"ShowKomposeBuildOption": false,
"SnapshotInterval": "5m",
"TemplatesURL": "",
"TrustOnFirstConnect": false,
"UserSessionTimeout": "8h",
"fdoConfiguration": {
"enabled": false,
"ownerPassword": "",
"ownerURL": "",
"ownerUsername": ""
},
"openAMTConfiguration": {
"certFileContent": "",
"certFileName": "",
@@ -776,6 +769,7 @@
"GpuUseList": null,
"HealthyContainerCount": 0,
"ImageCount": 9,
"IsPodman": false,
"NodeCount": 0,
"RunningContainerCount": 5,
"ServiceCount": 0,
@@ -810,7 +804,6 @@
"FromAppTemplate": false,
"GitConfig": null,
"Id": 2,
"IsComposeFormat": false,
"Name": "alpine",
"Namespace": "",
"Option": null,
@@ -833,7 +826,6 @@
"FromAppTemplate": false,
"GitConfig": null,
"Id": 5,
"IsComposeFormat": false,
"Name": "redis",
"Namespace": "",
"Option": null,
@@ -856,7 +848,6 @@
"FromAppTemplate": false,
"GitConfig": null,
"Id": 6,
"IsComposeFormat": false,
"Name": "nginx",
"Namespace": "",
"Option": null,

View File

@@ -52,27 +52,24 @@ func NewTestStore(t testing.TB, init, secure bool) (bool, *Store, func(), error)
}
if init {
err = store.Init()
if err != nil {
if err := store.Init(); err != nil {
return newStore, nil, nil, err
}
}
if newStore {
// from MigrateData
// From MigrateData
v := models.Version{
SchemaVersion: portainer.APIVersion,
Edition: int(portainer.PortainerCE),
}
err = store.VersionService.UpdateVersion(&v)
if err != nil {
if err := store.VersionService.UpdateVersion(&v); err != nil {
return newStore, nil, nil, err
}
}
teardown := func() {
err := store.Close()
if err != nil {
if err := store.Close(); err != nil {
log.Fatal().Err(err).Msg("")
}
}

View File

@@ -3,7 +3,7 @@ package consts
const (
ComposeStackNameLabel = "com.docker.compose.project"
SwarmStackNameLabel = "com.docker.stack.namespace"
SwarmServiceIdLabel = "com.docker.swarm.service.id"
SwarmNodeIdLabel = "com.docker.swarm.node.id"
SwarmServiceIDLabel = "com.docker.swarm.service.id"
SwarmNodeIDLabel = "com.docker.swarm.node.id"
HideStackLabel = "io.portainer.hideStack"
)

View File

@@ -4,15 +4,16 @@ import (
"context"
"strings"
"github.com/docker/docker/api/types"
dockercontainer "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
"github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/docker/images"
"github.com/Masterminds/semver/v3"
"github.com/docker/docker/api/types"
dockercontainer "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/network"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
@@ -30,16 +31,51 @@ func NewContainerService(factory *dockerclient.ClientFactory, dataStore dataserv
}
}
// applyVersionConstraint uses the version to apply a transformation function to
// the value when the constraint is satisfied
func applyVersionConstraint[T any](currentVersion, versionConstraint string, value T, transform func(T) T) (T, error) {
newValue := value
constraint, err := semver.NewConstraint(versionConstraint)
if err != nil {
return newValue, errors.New("invalid version constraint specified")
}
currentVer, err := semver.NewVersion(currentVersion)
if err != nil {
log.Warn().Err(err).Msg("Unable to parse the Docker client version")
return newValue, nil
}
if satisfiesConstraint, _ := constraint.Validate(currentVer); satisfiesConstraint {
newValue = transform(value)
}
return newValue, nil
}
func clearMacAddrs(n network.NetworkingConfig) network.NetworkingConfig {
netConfig := network.NetworkingConfig{
EndpointsConfig: make(map[string]*network.EndpointSettings),
}
for k := range n.EndpointsConfig {
endpointConfig := n.EndpointsConfig[k].Copy()
endpointConfig.MacAddress = ""
netConfig.EndpointsConfig[k] = endpointConfig
}
return netConfig
}
// Recreate a container
func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.Endpoint, containerId string, forcePullImage bool, imageTag, nodeName string) (*types.ContainerJSON, error) {
cli, err := c.factory.CreateClient(endpoint, nodeName, nil)
if err != nil {
return nil, errors.Wrap(err, "create client error")
}
defer func(cli *client.Client) {
cli.Close()
}(cli)
defer cli.Close()
log.Debug().Str("container_id", containerId).Msg("starting to fetch container information")
@@ -57,8 +93,7 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
}
if imageTag != "" {
err = img.WithTag(imageTag)
if err != nil {
if err := img.WithTag(imageTag); err != nil {
return nil, errors.Wrapf(err, "set image tag error %s", imageTag)
}
@@ -70,43 +105,39 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
// 1. pull image if you need force pull
if forcePullImage {
puller := images.NewPuller(cli, images.NewRegistryClient(c.dataStore), c.dataStore)
err = puller.Pull(ctx, img)
if err != nil {
if err := puller.Pull(ctx, img); err != nil {
return nil, errors.Wrapf(err, "pull image error %s", img.FullName())
}
}
// 2. stop the current container
log.Debug().Str("container_id", containerId).Msg("starting to stop the container")
err = cli.ContainerStop(ctx, containerId, dockercontainer.StopOptions{})
if err != nil {
if err := cli.ContainerStop(ctx, containerId, dockercontainer.StopOptions{}); err != nil {
return nil, errors.Wrap(err, "stop container error")
}
// 3. rename the current container
log.Debug().Str("container_id", containerId).Msg("starting to rename the container")
err = cli.ContainerRename(ctx, containerId, container.Name+"-old")
if err != nil {
if err := cli.ContainerRename(ctx, containerId, container.Name+"-old"); err != nil {
return nil, errors.Wrap(err, "rename container error")
}
networkWithCreation := network.NetworkingConfig{
initialNetwork := network.NetworkingConfig{
EndpointsConfig: make(map[string]*network.EndpointSettings),
}
// 4. disconnect all networks from the current container
for name, network := range container.NetworkSettings.Networks {
// This allows new container to use the same IP address if specified
err = cli.NetworkDisconnect(ctx, network.NetworkID, containerId, true)
if err != nil {
if err := cli.NetworkDisconnect(ctx, network.NetworkID, containerId, true); err != nil {
return nil, errors.Wrap(err, "disconnect network from old container error")
}
// 5. get the first network attached to the current container
if len(networkWithCreation.EndpointsConfig) == 0 {
if len(initialNetwork.EndpointsConfig) == 0 {
// Retrieve the first network that is linked to the present container, which
// will be utilized when creating the container.
networkWithCreation.EndpointsConfig[name] = network
initialNetwork.EndpointsConfig[name] = network
}
}
c.sr.enable()
@@ -116,9 +147,11 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
c.sr.push(func() {
log.Debug().Str("container_id", containerId).Str("container", container.Name).Msg("restoring the container")
cli.ContainerRename(ctx, containerId, container.Name)
for _, network := range container.NetworkSettings.Networks {
cli.NetworkConnect(ctx, network.NetworkID, containerId, network)
}
cli.ContainerStart(ctx, containerId, dockercontainer.StartOptions{})
})
@@ -130,7 +163,15 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
// to retain the same network settings we have to connect on creation to one of the old
// container's networks, and connect to the other networks after creation.
// see: https://portainer.atlassian.net/browse/EE-5448
create, err := cli.ContainerCreate(ctx, container.Config, container.HostConfig, &networkWithCreation, nil, container.Name)
// Docker API < 1.44 does not support specifying MAC addresses
// https://github.com/moby/moby/blob/6aea26b431ea152a8b085e453da06ea403f89886/client/container_create.go#L44-L46
initialNetwork, err = applyVersionConstraint(cli.ClientVersion(), "< 1.44", initialNetwork, clearMacAddrs)
if err != nil {
return nil, err
}
create, err := cli.ContainerCreate(ctx, container.Config, container.HostConfig, &initialNetwork, nil, container.Name)
c.sr.push(func() {
log.Debug().Str("container_id", create.ID).Msg("removing the new container")
@@ -150,22 +191,19 @@ func (c *ContainerService) Recreate(ctx context.Context, endpoint *portainer.End
log.Debug().Str("container_id", newContainerId).Msg("connecting networks to container")
networks := container.NetworkSettings.Networks
for key, network := range networks {
_, ok := networkWithCreation.EndpointsConfig[key]
if ok {
if _, ok := initialNetwork.EndpointsConfig[key]; ok {
// skip the network that is used during container creation
continue
}
err = cli.NetworkConnect(ctx, network.NetworkID, newContainerId, network)
if err != nil {
if err := cli.NetworkConnect(ctx, network.NetworkID, newContainerId, network); err != nil {
return nil, errors.Wrap(err, "connect container network error")
}
}
// 8. start the new container
log.Debug().Str("container_id", newContainerId).Msg("starting the new container")
err = cli.ContainerStart(ctx, newContainerId, dockercontainer.StartOptions{})
if err != nil {
if err := cli.ContainerStart(ctx, newContainerId, dockercontainer.StartOptions{}); err != nil {
return nil, errors.Wrap(err, "start container error")
}

View File

@@ -0,0 +1,52 @@
package docker
import (
"testing"
"github.com/docker/docker/api/types/network"
"github.com/stretchr/testify/require"
)
func TestApplyVersionConstraint(t *testing.T) {
initialNet := network.NetworkingConfig{
EndpointsConfig: map[string]*network.EndpointSettings{
"key1": {
MacAddress: "mac1",
EndpointID: "endpointID1",
},
"key2": {
MacAddress: "mac2",
EndpointID: "endpointID2",
},
},
}
f := func(currentVer string, constraint string, success, emptyMac bool) {
t.Helper()
transformedNet, err := applyVersionConstraint(currentVer, constraint, initialNet, clearMacAddrs)
if success {
require.NoError(t, err)
} else {
require.Error(t, err)
}
require.Len(t, transformedNet.EndpointsConfig, len(initialNet.EndpointsConfig))
for k := range initialNet.EndpointsConfig {
if emptyMac {
require.NotEqual(t, initialNet.EndpointsConfig[k], transformedNet.EndpointsConfig[k])
require.Empty(t, transformedNet.EndpointsConfig[k].MacAddress)
continue
}
require.Equal(t, initialNet.EndpointsConfig[k], transformedNet.EndpointsConfig[k])
}
}
f("1.45", "< 1.44", true, false) // No transformation
f("1.43", "< 1.44", true, true) // Transformation
f("a.b.", "< 1.44", true, false) // Invalid current version
f("1.45", "z 1.44", false, false) // Invalid version constraint
}

View File

@@ -1,6 +1,7 @@
package images
import (
"cmp"
"strings"
"time"
@@ -41,8 +42,7 @@ func (c *RegistryClient) RegistryAuth(image Image) (string, string, error) {
}
func (c *RegistryClient) CertainRegistryAuth(registry *portainer.Registry) (string, string, error) {
err := registryutils.EnsureRegTokenValid(c.dataStore, registry)
if err != nil {
if err := registryutils.EnsureRegTokenValid(c.dataStore, registry); err != nil {
return "", "", err
}
@@ -72,8 +72,7 @@ func (c *RegistryClient) EncodedRegistryAuth(image Image) (string, error) {
}
func (c *RegistryClient) EncodedCertainRegistryAuth(registry *portainer.Registry) (string, error) {
err := registryutils.EnsureRegTokenValid(c.dataStore, registry)
if err != nil {
if err := registryutils.EnsureRegTokenValid(c.dataStore, registry); err != nil {
return "", err
}
@@ -92,38 +91,32 @@ func findBestMatchRegistry(repository string, registries []portainer.Registry) (
}
var match1, match2, match3 *portainer.Registry
for i := 0; i < len(registries); i++ {
registry := registries[i]
if registry.Type == portainer.DockerHubRegistry {
// try to match repository examples:
// <USERNAME>/nginx:latest
// docker.io/<USERNAME>/nginx:latest
if strings.HasPrefix(repository, registry.Username+"/") || strings.HasPrefix(repository, registry.URL+"/"+registry.Username+"/") {
match1 = &registry
}
// try to match repository examples:
// portainer/portainer-ee:latest
// <NON-USERNAME>/portainer-ee:latest
if match3 == nil {
match3 = &registry
}
}
for _, registry := range registries {
if strings.Contains(repository, registry.URL) {
match2 = &registry
}
if registry.Type != portainer.DockerHubRegistry {
continue
}
// try to match repository examples:
// <USERNAME>/nginx:latest
// docker.io/<USERNAME>/nginx:latest
if strings.HasPrefix(repository, registry.Username+"/") || strings.HasPrefix(repository, registry.URL+"/"+registry.Username+"/") {
match1 = &registry
}
// try to match repository examples:
// portainer/portainer-ee:latest
// <NON-USERNAME>/portainer-ee:latest
if match3 == nil {
match3 = &registry
}
}
match := match1
if match == nil {
match = match2
}
if match == nil {
match = match3
}
match := cmp.Or(match1, match2, match3)
if match == nil {
return nil, errors.New("no registries matched")
}

View File

@@ -48,11 +48,11 @@ func (c *DigestClient) ContainersImageStatus(ctx context.Context, containers []t
statuses := make([]Status, len(containers))
for i, ct := range containers {
var nodeName string
if swarmNodeId := ct.Labels[consts.SwarmNodeIdLabel]; swarmNodeId != "" {
if swarmNodeId := ct.Labels[consts.SwarmNodeIDLabel]; swarmNodeId != "" {
if swarmNodeName, ok := swarmID2NameCache.Get(swarmNodeId); ok {
nodeName, _ = swarmNodeName.(string)
} else {
node, _, err := cli.NodeInspectWithRaw(ctx, ct.Labels[consts.SwarmNodeIdLabel])
node, _, err := cli.NodeInspectWithRaw(ctx, ct.Labels[consts.SwarmNodeIDLabel])
if err != nil {
return Error
}
@@ -160,7 +160,7 @@ func (c *DigestClient) ServiceImageStatus(ctx context.Context, serviceID string,
containers, err := cli.ContainerList(ctx, container.ListOptions{
All: true,
Filters: filters.NewArgs(filters.Arg("label", consts.SwarmServiceIdLabel+"="+serviceID)),
Filters: filters.NewArgs(filters.Arg("label", consts.SwarmServiceIDLabel+"="+serviceID)),
})
if err != nil {
log.Warn().Err(err).Str("serviceID", serviceID).Msg("cannot list container for the service")

View File

@@ -5,14 +5,15 @@ import (
"strings"
"time"
portainer "github.com/portainer/portainer/api"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/docker/consts"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
_container "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/volume"
"github.com/docker/docker/client"
portainer "github.com/portainer/portainer/api"
dockerclient "github.com/portainer/portainer/api/docker/client"
"github.com/portainer/portainer/api/docker/consts"
"github.com/rs/zerolog/log"
)
@@ -40,8 +41,7 @@ func (snapshotter *Snapshotter) CreateSnapshot(endpoint *portainer.Endpoint) (*p
}
func snapshot(cli *client.Client, endpoint *portainer.Endpoint) (*portainer.DockerSnapshot, error) {
_, err := cli.Ping(context.Background())
if err != nil {
if _, err := cli.Ping(context.Background()); err != nil {
return nil, err
}
@@ -49,49 +49,42 @@ func snapshot(cli *client.Client, endpoint *portainer.Endpoint) (*portainer.Dock
StackCount: 0,
}
err = snapshotInfo(snapshot, cli)
if err != nil {
if err := snapshotInfo(snapshot, cli); err != nil {
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot engine information")
}
if snapshot.Swarm {
err = snapshotSwarmServices(snapshot, cli)
if err != nil {
if err := snapshotSwarmServices(snapshot, cli); err != nil {
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot Swarm services")
}
err = snapshotNodes(snapshot, cli)
if err != nil {
if err := snapshotNodes(snapshot, cli); err != nil {
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot Swarm nodes")
}
}
err = snapshotContainers(snapshot, cli)
if err != nil {
if err := snapshotContainers(snapshot, cli); err != nil {
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot containers")
}
err = snapshotImages(snapshot, cli)
if err != nil {
if err := snapshotImages(snapshot, cli); err != nil {
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot images")
}
err = snapshotVolumes(snapshot, cli)
if err != nil {
if err := snapshotVolumes(snapshot, cli); err != nil {
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot volumes")
}
err = snapshotNetworks(snapshot, cli)
if err != nil {
if err := snapshotNetworks(snapshot, cli); err != nil {
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot networks")
}
err = snapshotVersion(snapshot, cli)
if err != nil {
if err := snapshotVersion(snapshot, cli); err != nil {
log.Warn().Str("environment", endpoint.Name).Err(err).Msg("unable to snapshot engine version")
}
snapshot.Time = time.Now().Unix()
return snapshot, nil
}
@@ -106,6 +99,7 @@ func snapshotInfo(snapshot *portainer.DockerSnapshot, cli *client.Client) error
snapshot.TotalCPU = info.NCPU
snapshot.TotalMemory = info.MemTotal
snapshot.SnapshotRaw.Info = info
return nil
}
@@ -114,15 +108,19 @@ func snapshotNodes(snapshot *portainer.DockerSnapshot, cli *client.Client) error
if err != nil {
return err
}
var nanoCpus int64
var totalMem int64
for _, node := range nodes {
nanoCpus += node.Description.Resources.NanoCPUs
totalMem += node.Description.Resources.MemoryBytes
}
snapshot.TotalCPU = int(nanoCpus / 1e9)
snapshot.TotalMemory = totalMem
snapshot.NodeCount = len(nodes)
return nil
}
@@ -144,6 +142,7 @@ func snapshotSwarmServices(snapshot *portainer.DockerSnapshot, cli *client.Clien
snapshot.ServiceCount = len(services)
snapshot.StackCount += len(stacks)
return nil
}
@@ -156,9 +155,10 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
stacks := make(map[string]struct{})
gpuUseSet := make(map[string]struct{})
gpuUseAll := false
for _, container := range containers {
if container.State == "running" {
// snapshot GPUs
// Snapshot GPUs
response, err := cli.ContainerInspect(context.Background(), container.ID)
if err != nil {
// Inspect a container will fail when the container runs on a different
@@ -177,7 +177,6 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
} else {
var gpuOptions *_container.DeviceRequest = nil
for _, deviceRequest := range response.HostConfig.Resources.DeviceRequests {
deviceRequest := deviceRequest
if deviceRequest.Driver == "nvidia" || deviceRequest.Capabilities[0][0] == "gpu" {
gpuOptions = &deviceRequest
}
@@ -187,6 +186,7 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
if gpuOptions.Count == -1 {
gpuUseAll = true
}
for _, id := range gpuOptions.DeviceIDs {
gpuUseSet[id] = struct{}{}
}
@@ -217,9 +217,11 @@ func snapshotContainers(snapshot *portainer.DockerSnapshot, cli *client.Client)
snapshot.HealthyContainerCount = stats.Healthy
snapshot.UnhealthyContainerCount = stats.Unhealthy
snapshot.StackCount += len(stacks)
for _, container := range containers {
snapshot.SnapshotRaw.Containers = append(snapshot.SnapshotRaw.Containers, portainer.DockerContainerSnapshot{Container: container})
}
return nil
}
@@ -231,6 +233,7 @@ func snapshotImages(snapshot *portainer.DockerSnapshot, cli *client.Client) erro
snapshot.ImageCount = len(images)
snapshot.SnapshotRaw.Images = images
return nil
}
@@ -242,6 +245,7 @@ func snapshotVolumes(snapshot *portainer.DockerSnapshot, cli *client.Client) err
snapshot.VolumeCount = len(volumes.Volumes)
snapshot.SnapshotRaw.Volumes = volumes
return nil
}
@@ -250,7 +254,9 @@ func snapshotNetworks(snapshot *portainer.DockerSnapshot, cli *client.Client) er
if err != nil {
return err
}
snapshot.SnapshotRaw.Networks = networks
return nil
}
@@ -259,6 +265,19 @@ func snapshotVersion(snapshot *portainer.DockerSnapshot, cli *client.Client) err
if err != nil {
return err
}
snapshot.SnapshotRaw.Version = version
snapshot.IsPodman = isPodman(version)
return nil
}
// isPodman checks if the version is for Podman by checking if any of the components contain "podman".
// If it's podman, a component name should be "Podman Engine"
func isPodman(version types.Version) bool {
for _, component := range version.Components {
if strings.Contains(strings.ToLower(component.Name), "podman") {
return true
}
}
return false
}

View File

@@ -38,7 +38,7 @@ func (manager *ComposeStackManager) ComposeSyntaxMaxVersion() string {
}
// Up builds, (re)creates and starts containers in the background. Wraps `docker-compose up -d` command
func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint, forceRecreate bool) error {
func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint, options portainer.ComposeUpOptions) error {
url, proxy, err := manager.fetchEndpointProxy(endpoint)
if err != nil {
return errors.Wrap(err, "failed to fetch environment proxy")
@@ -61,7 +61,39 @@ func (manager *ComposeStackManager) Up(ctx context.Context, stack *portainer.Sta
Host: url,
ProjectName: stack.Name,
},
ForceRecreate: forceRecreate,
ForceRecreate: options.ForceRecreate,
AbortOnContainerExit: options.AbortOnContainerExit,
})
return errors.Wrap(err, "failed to deploy a stack")
}
// Run runs a one-off command on a service. Wraps `docker-compose run` command
func (manager *ComposeStackManager) Run(ctx context.Context, stack *portainer.Stack, endpoint *portainer.Endpoint, serviceName string, options portainer.ComposeRunOptions) error {
url, proxy, err := manager.fetchEndpointProxy(endpoint)
if err != nil {
return errors.Wrap(err, "failed to fetch environment proxy")
}
if proxy != nil {
defer proxy.Close()
}
envFilePath, err := createEnvFile(stack)
if err != nil {
return errors.Wrap(err, "failed to create env file")
}
filePaths := stackutils.GetStackFilePaths(stack, true)
err = manager.deployer.Run(ctx, filePaths, serviceName, libstack.RunOptions{
Options: libstack.Options{
WorkingDir: stack.ProjectPath,
EnvFilePath: envFilePath,
Host: url,
ProjectName: stack.Name,
},
Remove: options.Remove,
Args: options.Args,
Detached: options.Detached,
})
return errors.Wrap(err, "failed to deploy a stack")
}

View File

@@ -60,7 +60,7 @@ func Test_UpAndDown(t *testing.T) {
ctx := context.TODO()
err = w.Up(ctx, stack, endpoint, false)
err = w.Up(ctx, stack, endpoint, portainer.ComposeUpOptions{})
if err != nil {
t.Fatalf("Error calling docker-compose up: %s", err)
}

View File

@@ -18,7 +18,3 @@ func (deployer *kubernetesMockDeployer) Deploy(userID portainer.UserID, endpoint
func (deployer *kubernetesMockDeployer) Remove(userID portainer.UserID, endpoint *portainer.Endpoint, manifestFiles []string, namespace string) (string, error) {
return "", nil
}
func (deployer *kubernetesMockDeployer) ConvertCompose(data []byte) ([]byte, error) {
return nil, nil
}

View File

@@ -44,7 +44,7 @@ func NewKubernetesDeployer(kubernetesTokenCacheManager *kubernetes.TokenCacheMan
}
func (deployer *KubernetesDeployer) getToken(userID portainer.UserID, endpoint *portainer.Endpoint, setLocalAdminToken bool) (string, error) {
kubeCLI, err := deployer.kubernetesClientFactory.GetKubeClient(endpoint)
kubeCLI, err := deployer.kubernetesClientFactory.GetPrivilegedKubeClient(endpoint)
if err != nil {
return "", err
}
@@ -137,29 +137,6 @@ func (deployer *KubernetesDeployer) command(operation string, userID portainer.U
return string(output), nil
}
// ConvertCompose leverages the kompose binary to deploy a compose compliant manifest.
func (deployer *KubernetesDeployer) ConvertCompose(data []byte) ([]byte, error) {
command := path.Join(deployer.binaryPath, "kompose")
if runtime.GOOS == "windows" {
command = path.Join(deployer.binaryPath, "kompose.exe")
}
args := make([]string, 0)
args = append(args, "convert", "-f", "-", "--stdout")
var stderr bytes.Buffer
cmd := exec.Command(command, args...)
cmd.Stderr = &stderr
cmd.Stdin = bytes.NewReader(data)
output, err := cmd.Output()
if err != nil {
return nil, errors.New(stderr.String())
}
return output, nil
}
func (deployer *KubernetesDeployer) getAgentURL(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) {
proxy, err := deployer.proxyManager.CreateAgentProxyServer(endpoint)
if err != nil {

View File

@@ -227,10 +227,10 @@ func (manager *SwarmStackManager) updateDockerCLIConfiguration(configPath string
}
if config["HttpHeaders"] == nil {
config["HttpHeaders"] = make(map[string]interface{})
config["HttpHeaders"] = make(map[string]any)
}
headersObject := config["HttpHeaders"].(map[string]interface{})
headersObject := config["HttpHeaders"].(map[string]any)
headersObject["X-PortainerAgent-ManagerOperation"] = "1"
headersObject["X-PortainerAgent-Signature"] = signature
headersObject["X-PortainerAgent-PublicKey"] = manager.signatureService.EncodedPublicKey()
@@ -238,12 +238,12 @@ func (manager *SwarmStackManager) updateDockerCLIConfiguration(configPath string
return manager.fileService.WriteJSONToFile(configFilePath, config)
}
func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (map[string]interface{}, error) {
var config map[string]interface{}
func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (map[string]any, error) {
var config map[string]any
raw, err := manager.fileService.GetFileContent(path, "")
if err != nil {
return make(map[string]interface{}), nil
return make(map[string]any), nil
}
err = json.Unmarshal(raw, &config)

View File

@@ -36,8 +36,6 @@ const (
ManifestFileDefaultName = "k8s-deployment.yml"
// EdgeStackStorePath represents the subfolder where edge stack files are stored in the file store folder.
EdgeStackStorePath = "edge_stacks"
// FDOProfileStorePath represents the subfolder where FDO profiles files are stored in the file store folder.
FDOProfileStorePath = "fdo_profiles"
// PrivateKeyFile represents the name on disk of the file containing the private key.
PrivateKeyFile = "portainer.key"
// PublicKeyFile represents the name on disk of the file containing the public key.
@@ -601,7 +599,7 @@ func (service *Service) Rename(oldPath, newPath string) error {
}
// WriteJSONToFile writes JSON to the specified file.
func (service *Service) WriteJSONToFile(path string, content interface{}) error {
func (service *Service) WriteJSONToFile(path string, content any) error {
jsonContent, err := json.Marshal(content)
if err != nil {
return err
@@ -1003,23 +1001,6 @@ func MoveDirectory(originalPath, newPath string, overwriteTargetPath bool) error
return os.Rename(originalPath, newPath)
}
// StoreFDOProfileFileFromBytes creates a subfolder in the FDOProfileStorePath and stores a new file from bytes.
// It returns the path to the folder where the file is stored.
func (service *Service) StoreFDOProfileFileFromBytes(fdoProfileIdentifier string, data []byte) (string, error) {
err := service.createDirectoryInStore(FDOProfileStorePath)
if err != nil {
return "", err
}
filePath := JoinPaths(FDOProfileStorePath, fdoProfileIdentifier)
err = service.createFileInStore(filePath, bytes.NewReader(data))
if err != nil {
return "", err
}
return service.wrapFileStore(filePath), nil
}
func CreateFile(path string, r io.Reader) error {
out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {

View File

@@ -6,7 +6,7 @@ import (
"path/filepath"
"strings"
"github.com/portainer/portainer/api"
portainer "github.com/portainer/portainer/api"
)
type MultiFilterArgs []struct {

View File

@@ -1,9 +1,10 @@
package filesystem
import (
"testing"
portainer "github.com/portainer/portainer/api"
"github.com/stretchr/testify/assert"
"testing"
)
func TestMultiFilterDirForPerDevConfigs(t *testing.T) {

View File

@@ -6,6 +6,8 @@ import (
"path/filepath"
"strings"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
@@ -14,7 +16,6 @@ import (
githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/storage/memory"
"github.com/pkg/errors"
gittypes "github.com/portainer/portainer/api/git/types"
)
type gitClient struct {
@@ -143,6 +144,7 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e
ReferenceName: plumbing.ReferenceName(opt.referenceName),
Auth: getAuth(opt.username, opt.password),
InsecureSkipTLS: opt.tlsSkipVerify,
Tags: git.NoTags,
}
repo, err := git.Clone(memory.NewStorage(), nil, cloneOption)
@@ -166,7 +168,10 @@ func (c *gitClient) listFiles(ctx context.Context, opt fetchOption) ([]string, e
}
var allPaths []string
w := object.NewTreeWalker(tree, true, nil)
defer w.Close()
for {
name, entry, err := w.Next()
if err != nil {

View File

@@ -91,6 +91,29 @@ func Test_latestCommitID(t *testing.T) {
assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id)
}
func Test_ListRefs(t *testing.T) {
service := Service{git: NewGitClient(true)}
repositoryURL := setup(t)
fs, err := service.ListRefs(repositoryURL, "", "", false, false)
assert.NoError(t, err)
assert.Equal(t, []string{"refs/heads/main"}, fs)
}
func Test_ListFiles(t *testing.T) {
service := Service{git: NewGitClient(true)}
repositoryURL := setup(t)
referenceName := "refs/heads/main"
fs, err := service.ListFiles(repositoryURL, referenceName, "", "", false, false, []string{".yml"}, false)
assert.NoError(t, err)
assert.Equal(t, []string{"docker-compose.yml"}, fs)
}
func getCommitHistoryLength(t *testing.T, err error, dir string) int {
repo, err := git.PlainOpen(dir)
if err != nil {

View File

@@ -9,6 +9,7 @@ import (
lru "github.com/hashicorp/golang-lru"
"github.com/rs/zerolog/log"
"golang.org/x/sync/singleflight"
)
const (
@@ -139,12 +140,18 @@ func (service *Service) CloneRepository(destination, repositoryURL, referenceNam
return service.cloneRepository(destination, options)
}
func (service *Service) cloneRepository(destination string, options cloneOption) error {
func (service *Service) repoManager(options baseOption) repoManager {
repoManager := service.git
if isAzureUrl(options.repositoryUrl) {
return service.azure.download(context.TODO(), destination, options)
repoManager = service.azure
}
return service.git.download(context.TODO(), destination, options)
return repoManager
}
func (service *Service) cloneRepository(destination string, options cloneOption) error {
return service.repoManager(options.baseOption).download(context.TODO(), destination, options)
}
// LatestCommitID returns SHA1 of the latest commit of the specified reference
@@ -159,11 +166,7 @@ func (service *Service) LatestCommitID(repositoryURL, referenceName, username, p
referenceName: referenceName,
}
if isAzureUrl(options.repositoryUrl) {
return service.azure.latestCommitID(context.TODO(), options)
}
return service.git.latestCommitID(context.TODO(), options)
return service.repoManager(options.baseOption).latestCommitID(context.TODO(), options)
}
// ListRefs will list target repository's references without cloning the repository
@@ -174,21 +177,16 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
service.repoRefCache.Remove(refCacheKey)
// Remove file caches pointed to the same repository
for _, fileCacheKey := range service.repoFileCache.Keys() {
key, ok := fileCacheKey.(string)
if ok {
if strings.HasPrefix(key, repositoryURL) {
service.repoFileCache.Remove(key)
}
if key, ok := fileCacheKey.(string); ok && strings.HasPrefix(key, repositoryURL) {
service.repoFileCache.Remove(key)
}
}
}
if service.repoRefCache != nil {
// Lookup the refs cache first
cache, ok := service.repoRefCache.Get(refCacheKey)
if ok {
refs, success := cache.([]string)
if success {
if cache, ok := service.repoRefCache.Get(refCacheKey); ok {
if refs, ok := cache.([]string); ok {
return refs, nil
}
}
@@ -201,33 +199,35 @@ func (service *Service) ListRefs(repositoryURL, username, password string, hardR
tlsSkipVerify: tlsSkipVerify,
}
var (
refs []string
err error
)
if isAzureUrl(options.repositoryUrl) {
refs, err = service.azure.listRefs(context.TODO(), options)
if err != nil {
return nil, err
}
} else {
refs, err = service.git.listRefs(context.TODO(), options)
if err != nil {
return nil, err
}
refs, err := service.repoManager(options).listRefs(context.TODO(), options)
if err != nil {
return nil, err
}
if service.cacheEnabled && service.repoRefCache != nil {
service.repoRefCache.Add(refCacheKey, refs)
}
return refs, nil
}
var singleflightGroup = &singleflight.Group{}
// ListFiles will list all the files of the target repository with specific extensions.
// If extension is not provided, it will list all the files under the target repository
func (service *Service) ListFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, includedExts []string, tlsSkipVerify bool) ([]string, error) {
repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly))
fs, err, _ := singleflightGroup.Do(repoKey, func() (any, error) {
return service.listFiles(repositoryURL, referenceName, username, password, dirOnly, hardRefresh, tlsSkipVerify)
})
return filterFiles(fs.([]string), includedExts), err
}
func (service *Service) listFiles(repositoryURL, referenceName, username, password string, dirOnly, hardRefresh bool, tlsSkipVerify bool) ([]string, error) {
repoKey := generateCacheKey(repositoryURL, referenceName, username, password, strconv.FormatBool(tlsSkipVerify), strconv.FormatBool(dirOnly))
if service.cacheEnabled && hardRefresh {
// Should remove the cache explicitly, so that the following normal list can show the correct result
service.repoFileCache.Remove(repoKey)
@@ -235,14 +235,9 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo
if service.repoFileCache != nil {
// lookup the files cache first
cache, ok := service.repoFileCache.Get(repoKey)
if ok {
files, success := cache.([]string)
if success {
// For the case while searching files in a repository without include extensions for the first time,
// but with include extensions for the second time
includedFiles := filterFiles(files, includedExts)
return includedFiles, nil
if cache, ok := service.repoFileCache.Get(repoKey); ok {
if files, ok := cache.([]string); ok {
return files, nil
}
}
}
@@ -258,28 +253,16 @@ func (service *Service) ListFiles(repositoryURL, referenceName, username, passwo
dirOnly: dirOnly,
}
var (
files []string
err error
)
if isAzureUrl(options.repositoryUrl) {
files, err = service.azure.listFiles(context.TODO(), options)
if err != nil {
return nil, err
}
} else {
files, err = service.git.listFiles(context.TODO(), options)
if err != nil {
return nil, err
}
files, err := service.repoManager(options.baseOption).listFiles(context.TODO(), options)
if err != nil {
return nil, err
}
includedFiles := filterFiles(files, includedExts)
if service.cacheEnabled && service.repoFileCache != nil {
service.repoFileCache.Add(repoKey, includedFiles)
return includedFiles, nil
service.repoFileCache.Add(repoKey, files)
}
return includedFiles, nil
return files, nil
}
func (service *Service) purgeCache() {
@@ -306,6 +289,7 @@ func matchExtensions(target string, exts []string) bool {
return true
}
}
return false
}
@@ -316,10 +300,11 @@ func filterFiles(paths []string, includedExts []string) []string {
var includedFiles []string
for _, filename := range paths {
// filter out the filenames with non-included extension
// Filter out the filenames with non-included extension
if matchExtensions(filename, includedExts) {
includedFiles = append(includedFiles, filename)
}
}
return includedFiles
}

View File

@@ -8,7 +8,7 @@ import (
)
func ValidateRepoConfig(repoConfig *gittypes.RepoConfig) error {
if govalidator.IsNull(repoConfig.URL) || !govalidator.IsURL(repoConfig.URL) {
if len(repoConfig.URL) == 0 || !govalidator.IsURL(repoConfig.URL) {
return httperrors.NewInvalidPayloadError("Invalid repository URL. Must correspond to a valid URL format")
}
@@ -17,7 +17,7 @@ func ValidateRepoConfig(repoConfig *gittypes.RepoConfig) error {
}
func ValidateRepoAuthentication(auth *gittypes.GitAuthentication) error {
if auth != nil && govalidator.IsNull(auth.Password) && auth.GitCredentialID == 0 {
if auth != nil && len(auth.Password) == 0 && auth.GitCredentialID == 0 {
return httperrors.NewInvalidPayloadError("Invalid repository credentials. Password or GitCredentialID must be specified when authentication is enabled")
}

View File

@@ -1,247 +0,0 @@
package fdo
import (
"bytes"
"errors"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/portainer/portainer/third_party/digest"
)
type FDOOwnerClient struct {
OwnerURL string
Username string
Password string
Timeout time.Duration
}
type ServiceInfo struct {
Module string
Var string
Filename string
Bytes []byte
GUID string
Device string
Priority int
OS string
Version string
Arch string
CRID int
Hash string
}
func (c FDOOwnerClient) doDigestAuthReq(method, endpoint, contentType string, body io.Reader) (*http.Response, error) {
transport := digest.NewTransport(c.Username, c.Password)
client, err := transport.Client()
if err != nil {
return nil, err
}
client.Timeout = c.Timeout
e, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
u, err := url.Parse(c.OwnerURL)
if err != nil {
return nil, err
}
req, err := http.NewRequest(method, u.ResolveReference(e).String(), body)
if err != nil {
return nil, err
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
return client.Do(req)
}
func (c FDOOwnerClient) PostVoucher(ov []byte) (string, error) {
resp, err := c.doDigestAuthReq(
http.MethodPost,
"api/v1/owner/vouchers",
"application/cbor",
bytes.NewReader(ov),
)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK {
return "", errors.New(http.StatusText(resp.StatusCode))
}
return string(body), nil
}
func (c FDOOwnerClient) PutDeviceSVI(info ServiceInfo) error {
values := url.Values{}
values.Set("module", info.Module)
values.Set("var", info.Var)
values.Set("filename", info.Filename)
values.Set("guid", info.GUID)
values.Set("device", info.Device)
values.Set("priority", strconv.Itoa(info.Priority))
values.Set("os", info.OS)
values.Set("version", info.Version)
values.Set("arch", info.Arch)
values.Set("crid", strconv.Itoa(info.CRID))
values.Set("hash", info.Hash)
resp, err := c.doDigestAuthReq(
http.MethodPut,
"api/v1/device/svi?"+values.Encode(),
"application/octet-stream",
strings.NewReader(string(info.Bytes)),
)
if err != nil {
return err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return errors.New(http.StatusText(resp.StatusCode))
}
return nil
}
func (c FDOOwnerClient) PutDeviceSVIRaw(info url.Values, body []byte) error {
resp, err := c.doDigestAuthReq(
http.MethodPut,
"api/v1/device/svi?"+info.Encode(),
"application/octet-stream",
strings.NewReader(string(body)),
)
if err != nil {
return err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return errors.New(http.StatusText(resp.StatusCode))
}
return nil
}
func (c FDOOwnerClient) GetVouchers() ([]string, error) {
resp, err := c.doDigestAuthReq(
http.MethodGet,
"api/v1/owner/vouchers",
"",
nil,
)
if err != nil {
return nil, err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, errors.New(http.StatusText(resp.StatusCode))
}
contents, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
guids := strings.FieldsFunc(
strings.TrimSpace(string(contents)),
func(c rune) bool {
return c == ','
},
)
return guids, nil
}
func (c FDOOwnerClient) DeleteVoucher(guid string) error {
resp, err := c.doDigestAuthReq(
http.MethodDelete,
"api/v1/owner/vouchers?id="+guid,
"",
nil,
)
if err != nil {
return err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return errors.New(http.StatusText(resp.StatusCode))
}
return nil
}
func (c FDOOwnerClient) GetDeviceSVI(guid string) (string, error) {
resp, err := c.doDigestAuthReq(
http.MethodGet,
"api/v1/device/svi?guid="+guid,
"",
nil,
)
if err != nil {
return "", err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK {
return "", errors.New(http.StatusText(resp.StatusCode))
}
return string(body), nil
}
func (c FDOOwnerClient) DeleteDeviceSVI(id string) error {
resp, err := c.doDigestAuthReq(
http.MethodDelete,
"api/v1/device/svi?id="+id,
"",
nil,
)
if err != nil {
return err
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return errors.New(http.StatusText(resp.StatusCode))
}
return nil
}

View File

@@ -12,7 +12,7 @@ import (
func (service *Service) enableDeviceFeatures(configuration portainer.OpenAMTConfiguration, deviceGUID string, features portainer.OpenAMTDeviceEnabledFeatures) error {
url := fmt.Sprintf("https://%s/mps/api/v1/amt/features/%s", configuration.MPSServer, deviceGUID)
payload := map[string]interface{}{
payload := map[string]any{
"enableSOL": features.SOL,
"enableIDER": features.IDER,
"enableKVM": features.KVM,

View File

@@ -4,20 +4,27 @@ import (
"crypto/rand"
"fmt"
"net/http"
"os"
"github.com/portainer/portainer/api/http/security"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
gorillacsrf "github.com/gorilla/csrf"
"github.com/portainer/portainer/api/http/security"
"github.com/urfave/negroni"
)
func WithProtect(handler http.Handler) (http.Handler, error) {
// IsDockerDesktopExtension is used to check if we should skip csrf checks in the request bouncer (ShouldSkipCSRFCheck)
// DOCKER_EXTENSION is set to '1' in build/docker-extension/docker-compose.yml
isDockerDesktopExtension := false
if val, ok := os.LookupEnv("DOCKER_EXTENSION"); ok && val == "1" {
isDockerDesktopExtension = true
}
handler = withSendCSRFToken(handler)
token := make([]byte, 32)
_, err := rand.Read(token)
if err != nil {
if _, err := rand.Read(token); err != nil {
return nil, fmt.Errorf("failed to generate CSRF token: %w", err)
}
@@ -27,12 +34,11 @@ func WithProtect(handler http.Handler) (http.Handler, error) {
gorillacsrf.Secure(false),
)(handler)
return withSkipCSRF(handler), nil
return withSkipCSRF(handler, isDockerDesktopExtension), nil
}
func withSendCSRFToken(handler http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
sw := negroni.NewResponseWriter(w)
sw.Before(func(sw negroni.ResponseWriter) {
@@ -44,16 +50,15 @@ func withSendCSRFToken(handler http.Handler) http.Handler {
})
handler.ServeHTTP(sw, r)
})
}
func withSkipCSRF(handler http.Handler) http.Handler {
func withSkipCSRF(handler http.Handler, isDockerDesktopExtension bool) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
skip, err := security.ShouldSkipCSRFCheck(r)
skip, err := security.ShouldSkipCSRFCheck(r, isDockerDesktopExtension)
if err != nil {
httperror.WriteError(w, http.StatusForbidden, err.Error(), err)
return
}

View File

@@ -12,7 +12,6 @@ import (
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/asaskevich/govalidator"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
@@ -30,11 +29,11 @@ type authenticateResponse struct {
}
func (payload *authenticatePayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.Username) {
if len(payload.Username) == 0 {
return errors.New("Invalid username")
}
if govalidator.IsNull(payload.Password) {
if len(payload.Password) == 0 {
return errors.New("Invalid password")
}
@@ -56,8 +55,7 @@ func (payload *authenticatePayload) Validate(r *http.Request) error {
// @router /auth [post]
func (handler *Handler) authenticate(rw http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload authenticatePayload
err := request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
@@ -104,8 +102,7 @@ func isUserInitialAdmin(user *portainer.User) bool {
}
func (handler *Handler) authenticateInternal(w http.ResponseWriter, user *portainer.User, password string) *httperror.HandlerError {
err := handler.CryptoService.CompareHashAndData(user.Password, password)
if err != nil {
if err := handler.CryptoService.CompareHashAndData(user.Password, password); err != nil {
return httperror.NewError(http.StatusUnprocessableEntity, "Invalid credentials", httperrors.ErrUnauthorized)
}
@@ -115,8 +112,7 @@ func (handler *Handler) authenticateInternal(w http.ResponseWriter, user *portai
}
func (handler *Handler) authenticateLDAP(w http.ResponseWriter, user *portainer.User, username, password string, ldapSettings *portainer.LDAPSettings) *httperror.HandlerError {
err := handler.LDAPService.AuthenticateUser(username, password, ldapSettings)
if err != nil {
if err := handler.LDAPService.AuthenticateUser(username, password, ldapSettings); err != nil {
if errors.Is(err, httperrors.ErrUnauthorized) {
return httperror.NewError(http.StatusUnprocessableEntity, "Invalid credentials", httperrors.ErrUnauthorized)
}
@@ -131,14 +127,12 @@ func (handler *Handler) authenticateLDAP(w http.ResponseWriter, user *portainer.
PortainerAuthorizations: authorization.DefaultPortainerAuthorizations(),
}
err = handler.DataStore.User().Create(user)
if err != nil {
if err := handler.DataStore.User().Create(user); err != nil {
return httperror.InternalServerError("Unable to persist user inside the database", err)
}
}
err = handler.syncUserTeamsWithLDAPGroups(user, ldapSettings)
if err != nil {
if err := handler.syncUserTeamsWithLDAPGroups(user, ldapSettings); err != nil {
log.Warn().Err(err).Msg("unable to automatically sync user teams with ldap")
}
@@ -160,7 +154,6 @@ func (handler *Handler) persistAndWriteToken(w http.ResponseWriter, tokenData *p
security.AddAuthCookie(w, token, expirationTime)
return response.JSON(w, &authenticateResponse{JWT: token})
}
func (handler *Handler) syncUserTeamsWithLDAPGroups(user *portainer.User, settings *portainer.LDAPSettings) error {
@@ -185,22 +178,18 @@ func (handler *Handler) syncUserTeamsWithLDAPGroups(user *portainer.User, settin
}
for _, team := range teams {
if teamExists(team.Name, userGroups) {
if !teamExists(team.Name, userGroups) || teamMembershipExists(team.ID, userMemberships) {
continue
}
if teamMembershipExists(team.ID, userMemberships) {
continue
}
membership := &portainer.TeamMembership{
UserID: user.ID,
TeamID: team.ID,
Role: portainer.TeamMember,
}
membership := &portainer.TeamMembership{
UserID: user.ID,
TeamID: team.ID,
Role: portainer.TeamMember,
}
err := handler.DataStore.TeamMembership().Create(membership)
if err != nil {
return err
}
if err := handler.DataStore.TeamMembership().Create(membership); err != nil {
return err
}
}

View File

@@ -9,7 +9,6 @@ import (
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/asaskevich/govalidator"
"github.com/rs/zerolog/log"
)
@@ -19,7 +18,7 @@ type oauthPayload struct {
}
func (payload *oauthPayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.Code) {
if len(payload.Code) == 0 {
return errors.New("Invalid OAuth authorization code")
}

View File

@@ -41,5 +41,6 @@ func NewHandler(bouncer security.BouncerService, rateLimiter *security.RateLimit
rateLimiter.LimitAccess(bouncer.PublicAccess(httperror.LoggerHandler(h.authenticate)))).Methods(http.MethodPost)
h.Handle("/auth/logout",
bouncer.PublicAccess(httperror.LoggerHandler(h.logout))).Methods(http.MethodPost)
return h
}

View File

@@ -4,7 +4,7 @@ import (
"net/http"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/logoutcontext"
"github.com/portainer/portainer/api/logoutcontext"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/response"
)
@@ -28,5 +28,7 @@ func (handler *Handler) logout(w http.ResponseWriter, r *http.Request) *httperro
security.RemoveAuthCookie(w)
handler.bouncer.RevokeJWT(tokenData.Token)
return response.Empty(w)
}

View File

@@ -108,13 +108,13 @@ type customTemplateFromFileContentPayload struct {
}
func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.Title) {
if len(payload.Title) == 0 {
return errors.New("Invalid custom template title")
}
if govalidator.IsNull(payload.Description) {
if len(payload.Description) == 0 {
return errors.New("Invalid custom template description")
}
if govalidator.IsNull(payload.FileContent) {
if len(payload.FileContent) == 0 {
return errors.New("Invalid file content")
}
if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
@@ -132,7 +132,7 @@ func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) e
}
func isValidNote(note string) bool {
if govalidator.IsNull(note) {
if len(note) == 0 {
return true
}
match, _ := regexp.MatchString("<img", note)
@@ -226,19 +226,19 @@ type customTemplateFromGitRepositoryPayload struct {
}
func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.Title) {
if len(payload.Title) == 0 {
return errors.New("Invalid custom template title")
}
if govalidator.IsNull(payload.Description) {
if len(payload.Description) == 0 {
return errors.New("Invalid custom template description")
}
if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) {
if len(payload.RepositoryURL) == 0 || !govalidator.IsURL(payload.RepositoryURL) {
return errors.New("Invalid repository URL. Must correspond to a valid URL format")
}
if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
if payload.RepositoryAuthentication && (len(payload.RepositoryUsername) == 0 || len(payload.RepositoryPassword) == 0) {
return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
}
if govalidator.IsNull(payload.ComposeFilePathInRepository) {
if len(payload.ComposeFilePathInRepository) == 0 {
payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
}

View File

@@ -70,8 +70,7 @@ func (handler *Handler) customTemplateGitFetch(w http.ResponseWriter, r *http.Re
if err != nil {
log.Warn().Err(err).Msg("failed to download git repository")
if err != nil {
rbErr := rollbackCustomTemplate(backupPath, customTemplate.ProjectPath)
if rbErr := rollbackCustomTemplate(backupPath, customTemplate.ProjectPath); rbErr != nil {
return httperror.InternalServerError("Failed to rollback the custom template folder", rbErr)
}

View File

@@ -2,6 +2,7 @@ package customtemplates
import (
"bytes"
"errors"
"io"
"io/fs"
"net/http"
@@ -19,6 +20,7 @@ import (
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/internal/testhelpers"
"github.com/portainer/portainer/api/jwt"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/segmentio/encoding/json"
"github.com/stretchr/testify/assert"
@@ -49,6 +51,19 @@ func (f *TestFileService) GetFileContent(projectPath, configFilePath string) ([]
return os.ReadFile(filepath.Join(projectPath, configFilePath))
}
type InvalidTestGitService struct {
portainer.GitService
targetFilePath string
}
func (g *InvalidTestGitService) CloneRepository(dest, repoUrl, refName, username, password string, tlsSkipVerify bool) error {
return errors.New("simulate network error")
}
func (g *InvalidTestGitService) LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) {
return "", nil
}
func createTestFile(targetPath string) error {
f, err := os.Create(targetPath)
if err != nil {
@@ -75,7 +90,7 @@ func singleAPIRequest(h *Handler, jwt string, is *assert.Assertions, expect stri
FileContent string
}
req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBuffer([]byte("{}")))
req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBufferString("{}"))
testhelpers.AddTestSecurityCookie(req, jwt)
rr := httptest.NewRecorder()
@@ -142,28 +157,33 @@ func Test_customTemplateGitFetch(t *testing.T) {
t.Run("can return the expected file content by multiple calls from one user", func(t *testing.T) {
var wg sync.WaitGroup
wg.Add(5)
for i := 0; i < 5; i++ {
for range 5 {
go func() {
singleAPIRequest(h, jwt1, is, "abcdefg")
wg.Done()
}()
}
wg.Wait()
})
t.Run("can return the expected file content by multiple calls from different users", func(t *testing.T) {
var wg sync.WaitGroup
wg.Add(10)
for i := 0; i < 10; i++ {
for i := range 10 {
go func(j int) {
if j%2 == 0 {
singleAPIRequest(h, jwt1, is, "abcdefg")
} else {
singleAPIRequest(h, jwt2, is, "abcdefg")
}
wg.Done()
}(i)
}
wg.Wait()
})
@@ -174,4 +194,28 @@ func Test_customTemplateGitFetch(t *testing.T) {
singleAPIRequest(h, jwt2, is, "gfedcba")
})
t.Run("restore git repository if it is failed to download the new git repository", func(t *testing.T) {
invalidGitService := &InvalidTestGitService{
targetFilePath: filepath.Join(template1.ProjectPath, template1.GitConfig.ConfigFilePath),
}
h := NewHandler(requestBouncer, store, fileService, invalidGitService)
req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBufferString("{}"))
testhelpers.AddTestSecurityCookie(req, jwt1)
rr := httptest.NewRecorder()
h.ServeHTTP(rr, req)
is.Equal(http.StatusInternalServerError, rr.Code)
var errResp httperror.HandlerError
err = json.NewDecoder(rr.Body).Decode(&errResp)
assert.NoError(t, err, "failed to parse error body")
assert.FileExists(t, gitService.targetFilePath, "previous git repository is not restored")
fileContent, err := os.ReadFile(gitService.targetFilePath)
assert.NoError(t, err, "failed to read target file")
assert.Equal(t, "gfedcba", string(fileContent))
})
}

View File

@@ -4,14 +4,15 @@ import (
"net/http"
"strconv"
"github.com/pkg/errors"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/internal/slices"
"github.com/portainer/portainer/api/slicesx"
httperror "github.com/portainer/portainer/pkg/libhttp/error"
"github.com/portainer/portainer/pkg/libhttp/request"
"github.com/portainer/portainer/pkg/libhttp/response"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
@@ -70,7 +71,7 @@ func (handler *Handler) customTemplateList(w http.ResponseWriter, r *http.Reques
customTemplates = filterByType(customTemplates, templateTypes)
if edge != nil {
customTemplates = slices.Filter(customTemplates, func(customTemplate portainer.CustomTemplate) bool {
customTemplates = slicesx.Filter(customTemplates, func(customTemplate portainer.CustomTemplate) bool {
return customTemplate.EdgeTemplate == *edge
})
}

View File

@@ -64,34 +64,39 @@ type customTemplateUpdatePayload struct {
}
func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.Title) {
if len(payload.Title) == 0 {
return errors.New("Invalid custom template title")
}
if govalidator.IsNull(payload.FileContent) && govalidator.IsNull(payload.RepositoryURL) {
if len(payload.FileContent) == 0 && len(payload.RepositoryURL) == 0 {
return errors.New("Either file content or git repository url need to be provided")
}
if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
return errors.New("Invalid custom template platform")
}
if payload.Type != portainer.KubernetesStack && payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
return errors.New("Invalid custom template type")
}
if govalidator.IsNull(payload.Description) {
if len(payload.Description) == 0 {
return errors.New("Invalid custom template description")
}
if !isValidNote(payload.Note) {
return errors.New("Invalid note. <img> tag is not supported")
}
if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
if payload.RepositoryAuthentication && (len(payload.RepositoryUsername) == 0 || len(payload.RepositoryPassword) == 0) {
return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
}
if govalidator.IsNull(payload.ComposeFilePathInRepository) {
if len(payload.ComposeFilePathInRepository) == 0 {
payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
}
err := validateVariablesDefinitions(payload.Variables)
if err != nil {
if err := validateVariablesDefinitions(payload.Variables); err != nil {
return err
}
@@ -122,8 +127,7 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
}
var payload customTemplateUpdatePayload
err = request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
@@ -219,8 +223,7 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
customTemplate.ProjectPath = projectPath
}
err = handler.DataStore.CustomTemplate().Update(customTemplate.ID, customTemplate)
if err != nil {
if err := handler.DataStore.CustomTemplate().Update(customTemplate.ID, customTemplate); err != nil {
return httperror.InternalServerError("Unable to persist custom template changes inside the database", err)
}

Some files were not shown because too many files have changed in this diff Show More