Compare commits

...

10 Commits

29 changed files with 312 additions and 179 deletions

View File

@@ -93,6 +93,8 @@ body:
description: We only provide support for the most recent version of Portainer and the previous 3 versions. If you are on an older version of Portainer we recommend [upgrading first](https://docs.portainer.io/start/upgrade) in case your bug has already been fixed.
multiple: false
options:
- '2.22.0'
- '2.21.3'
- '2.21.2'
- '2.21.1'
- '2.21.0'

View File

@@ -2,6 +2,7 @@ package edgestacks
import (
"errors"
"fmt"
"net/http"
"time"
@@ -63,7 +64,7 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req
var payload updateStatusPayload
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", err)
return httperror.BadRequest("Invalid request payload", fmt.Errorf("edge polling error: %w. Environment ID: %d", err, payload.EndpointID))
}
var stack *portainer.EdgeStack
@@ -95,16 +96,16 @@ func (handler *Handler) updateEdgeStackStatus(tx dataservices.DataStoreTx, r *ht
return nil, nil
}
return nil, err
return nil, fmt.Errorf("unable to retrieve Edge stack from the database: %w. Environment ID: %d", err, payload.EndpointID)
}
endpoint, err := tx.Endpoint().Endpoint(payload.EndpointID)
if err != nil {
return nil, handler.handlerDBErr(err, "Unable to find an environment with the specified identifier inside the database")
return nil, handler.handlerDBErr(fmt.Errorf("unable to find the environment from the database: %w. Environment ID: %d", err, payload.EndpointID), "unable to find the environment")
}
if err := handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint); err != nil {
return nil, httperror.Forbidden("Permission denied to access environment", err)
return nil, httperror.Forbidden("Permission denied to access environment", fmt.Errorf("unauthorized edge endpoint operation: %w. Environment name: %s", err, endpoint.Name))
}
status := *payload.Status
@@ -123,7 +124,7 @@ func (handler *Handler) updateEdgeStackStatus(tx dataservices.DataStoreTx, r *ht
updateEnvStatus(payload.EndpointID, stack, deploymentStatus)
if err := tx.EdgeStack().UpdateEdgeStack(stackID, stack); err != nil {
return nil, handler.handlerDBErr(err, "Unable to persist the stack changes inside the database")
return nil, handler.handlerDBErr(fmt.Errorf("unable to update Edge stack to the database: %w. Environment name: %s", err, endpoint.Name), "unable to update Edge stack")
}
return stack, nil

View File

@@ -2,6 +2,7 @@ package endpointedge
import (
"errors"
"fmt"
"net/http"
"strconv"
@@ -39,32 +40,30 @@ func (handler *Handler) endpointEdgeJobsLogs(w http.ResponseWriter, r *http.Requ
return httperror.BadRequest("Unable to find an environment on request context", err)
}
err = handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint)
if err != nil {
return httperror.Forbidden("Permission denied to access environment", err)
if err := handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint); err != nil {
return httperror.Forbidden("Permission denied to access environment", fmt.Errorf("unauthorized edge endpoint operation: %w. Environment name: %s", err, endpoint.Name))
}
edgeJobID, err := request.RetrieveNumericRouteVariableValue(r, "jobID")
if err != nil {
return httperror.BadRequest("Invalid edge job identifier route variable", err)
return httperror.BadRequest("Invalid edge job identifier route variable", fmt.Errorf("invalid Edge job route variable: %w. Environment name: %s", err, endpoint.Name))
}
var payload logsPayload
err = request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return httperror.BadRequest("Invalid request payload", err)
if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
return httperror.BadRequest("Invalid request payload", fmt.Errorf("invalid Edge job request payload: %w. Environment name: %s", err, endpoint.Name))
}
err = handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
if err := handler.DataStore.UpdateTx(func(tx dataservices.DataStoreTx) error {
return handler.getEdgeJobLobs(tx, endpoint.ID, portainer.EdgeJobID(edgeJobID), payload)
})
if err != nil {
}); err != nil {
var httpErr *httperror.HandlerError
if errors.As(err, &httpErr) {
httpErr.Err = fmt.Errorf("edge polling error: %w. Environment name: %s", httpErr.Err, endpoint.Name)
return httpErr
}
return httperror.InternalServerError("Unexpected error", err)
return httperror.InternalServerError("Unexpected error", fmt.Errorf("edge polling error: %w. Environment name: %s", err, endpoint.Name))
}
return response.JSON(w, nil)

View File

@@ -1,7 +1,7 @@
package endpointedge
import (
"errors"
"fmt"
"net/http"
portainer "github.com/portainer/portainer/api"
@@ -33,27 +33,26 @@ func (handler *Handler) endpointEdgeStackInspect(w http.ResponseWriter, r *http.
return httperror.BadRequest("Unable to find an environment on request context", err)
}
err = handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint)
if err != nil {
return httperror.Forbidden("Permission denied to access environment", err)
if err := handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint); err != nil {
return httperror.Forbidden("Permission denied to access environment", fmt.Errorf("unauthorized edge endpoint operation: %w. Environment name: %s", err, endpoint.Name))
}
edgeStackID, err := request.RetrieveNumericRouteVariableValue(r, "stackId")
if err != nil {
return httperror.BadRequest("Invalid edge stack identifier route variable", err)
return httperror.BadRequest("Invalid edge stack identifier route variable", fmt.Errorf("invalid Edge stack route variable: %w. Environment name: %s", err, endpoint.Name))
}
edgeStack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(edgeStackID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find an edge stack with the specified identifier inside the database", err)
return httperror.NotFound("Unable to find an edge stack with the specified identifier inside the database", fmt.Errorf("unable to find the Edge stack from database: %w. Environment name: %s", err, endpoint.Name))
} else if err != nil {
return httperror.InternalServerError("Unable to find an edge stack with the specified identifier inside the database", err)
return httperror.InternalServerError("Unable to find an edge stack with the specified identifier inside the database", fmt.Errorf("failed to find the Edge stack from database: %w. Environment name: %s", err, endpoint.Name))
}
fileName := edgeStack.EntryPoint
if endpointutils.IsDockerEndpoint(endpoint) {
if fileName == "" {
return httperror.BadRequest("Docker is not supported by this stack", errors.New("Docker is not supported by this stack"))
return httperror.BadRequest("Docker is not supported by this stack", fmt.Errorf("no filename is provided for the Docker endpoint. Environment name: %s", endpoint.Name))
}
}
@@ -66,18 +65,18 @@ func (handler *Handler) endpointEdgeStackInspect(w http.ResponseWriter, r *http.
fileName = edgeStack.ManifestPath
if fileName == "" {
return httperror.BadRequest("Kubernetes is not supported by this stack", errors.New("Kubernetes is not supported by this stack"))
return httperror.BadRequest("Kubernetes is not supported by this stack", fmt.Errorf("no filename is provided for the Kubernetes endpoint. Environment name: %s", endpoint.Name))
}
}
dirEntries, err := filesystem.LoadDir(edgeStack.ProjectPath)
if err != nil {
return httperror.InternalServerError("Unable to load repository", err)
return httperror.InternalServerError("Unable to load repository", fmt.Errorf("failed to load project directory: %w. Environment name: %s", err, endpoint.Name))
}
fileContent, err := filesystem.FilterDirForCompatibility(dirEntries, fileName, endpoint.Agent.Version)
if err != nil {
return httperror.InternalServerError("File not found", err)
return httperror.InternalServerError("File not found", fmt.Errorf("unable to find file: %w. Environment name: %s", err, endpoint.Name))
}
dirEntries = filesystem.FilterDirForEntryFile(dirEntries, fileName)

View File

@@ -85,25 +85,25 @@ func (handler *Handler) endpointEdgeStatusInspect(w http.ResponseWriter, r *http
if _, ok := handler.DataStore.Endpoint().Heartbeat(portainer.EndpointID(endpointID)); !ok {
// EE-5190
return httperror.Forbidden("Permission denied to access environment", errors.New("the device has not been trusted yet"))
return httperror.Forbidden("Permission denied to access environment. The device has not been trusted yet", fmt.Errorf("unable to retrieve endpoint heartbeat. Environment ID: %d", endpointID))
}
endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
if err != nil {
// EE-5190
return httperror.Forbidden("Permission denied to access environment", errors.New("the device has not been trusted yet"))
return httperror.Forbidden("Permission denied to access environment. The device has not been trusted yet", fmt.Errorf("unable to retrieve endpoint from database: %w. Environment ID: %d", err, endpointID))
}
firstConn := endpoint.LastCheckInDate == 0
if err := handler.requestBouncer.AuthorizedEdgeEndpointOperation(r, endpoint); err != nil {
return httperror.Forbidden("Permission denied to access environment", err)
return httperror.Forbidden("Permission denied to access environment. The device has not been trusted yet", fmt.Errorf("unauthorized Edge endpoint operation: %w. Environment name: %s", err, endpoint.Name))
}
handler.DataStore.Endpoint().UpdateHeartbeat(endpoint.ID)
if err := handler.requestBouncer.TrustedEdgeEnvironmentAccess(handler.DataStore, endpoint); err != nil {
return httperror.Forbidden("Permission denied to access environment", err)
return httperror.Forbidden("Permission denied to access environment. The device has not been trusted yet", fmt.Errorf("untrusted Edge environment access: %w. Environment name: %s", err, endpoint.Name))
}
var statusResponse *endpointEdgeStatusInspectResponse
@@ -113,10 +113,11 @@ func (handler *Handler) endpointEdgeStatusInspect(w http.ResponseWriter, r *http
}); err != nil {
var httpErr *httperror.HandlerError
if errors.As(err, &httpErr) {
httpErr.Err = fmt.Errorf("edge polling error: %w. Environment name: %s", httpErr.Err, endpoint.Name)
return httpErr
}
return httperror.InternalServerError("Unexpected error", err)
return httperror.InternalServerError("Unexpected error", fmt.Errorf("edge polling error: %w. Environment name: %s", err, endpoint.Name))
}
return cacheResponse(w, endpoint.ID, *statusResponse)

View File

@@ -98,8 +98,8 @@ func (handler *Handler) updateEndpointGroup(tx dataservices.DataStoreTx, endpoin
payloadTagSet := tag.Set(payload.TagIDs)
endpointGroupTagSet := tag.Set((endpointGroup.TagIDs))
union := tag.Union(payloadTagSet, endpointGroupTagSet)
intersection := tag.Intersection(payloadTagSet, endpointGroupTagSet)
tagsChanged = len(union) > len(intersection)
intersection := tag.IntersectionCount(payloadTagSet, endpointGroupTagSet)
tagsChanged = len(union) > intersection
if tagsChanged {
removeTags := tag.Difference(endpointGroupTagSet, payloadTagSet)

View File

@@ -193,7 +193,7 @@ func (handler *Handler) filterEndpointsByQuery(
return nil, 0, errors.WithMessage(err, "Unable to retrieve tags from the database")
}
tagsMap := make(map[portainer.TagID]string)
tagsMap := make(map[portainer.TagID]string, len(tags))
for _, tag := range tags {
tagsMap[tag.ID] = tag.Name
}
@@ -304,8 +304,7 @@ func filterEndpointsBySearchCriteria(
) []portainer.Endpoint {
n := 0
for _, endpoint := range endpoints {
endpointTags := convertTagIDsToTags(tagsMap, endpoint.TagIDs)
if endpointMatchSearchCriteria(&endpoint, endpointTags, searchCriteria) {
if endpointMatchSearchCriteria(&endpoint, tagsMap, searchCriteria) {
endpoints[n] = endpoint
n++
@@ -319,7 +318,7 @@ func filterEndpointsBySearchCriteria(
continue
}
if edgeGroupMatchSearchCriteria(&endpoint, edgeGroups, searchCriteria, endpoints, endpointGroups) {
if edgeGroupMatchSearchCriteria(&endpoint, edgeGroups, searchCriteria, endpointGroups) {
endpoints[n] = endpoint
n++
@@ -365,7 +364,7 @@ func filterEndpointsByStatuses(endpoints []portainer.Endpoint, statuses []portai
return endpoints[:n]
}
func endpointMatchSearchCriteria(endpoint *portainer.Endpoint, tags []string, searchCriteria string) bool {
func endpointMatchSearchCriteria(endpoint *portainer.Endpoint, tagsMap map[portainer.TagID]string, searchCriteria string) bool {
if strings.Contains(strings.ToLower(endpoint.Name), searchCriteria) {
return true
}
@@ -380,8 +379,8 @@ func endpointMatchSearchCriteria(endpoint *portainer.Endpoint, tags []string, se
return true
}
for _, tag := range tags {
if strings.Contains(strings.ToLower(tag), searchCriteria) {
for _, tagID := range endpoint.TagIDs {
if strings.Contains(strings.ToLower(tagsMap[tagID]), searchCriteria) {
return true
}
}
@@ -391,16 +390,17 @@ func endpointMatchSearchCriteria(endpoint *portainer.Endpoint, tags []string, se
func endpointGroupMatchSearchCriteria(endpoint *portainer.Endpoint, endpointGroups []portainer.EndpointGroup, tagsMap map[portainer.TagID]string, searchCriteria string) bool {
for _, group := range endpointGroups {
if group.ID == endpoint.GroupID {
if strings.Contains(strings.ToLower(group.Name), searchCriteria) {
return true
}
if group.ID != endpoint.GroupID {
continue
}
tags := convertTagIDsToTags(tagsMap, group.TagIDs)
for _, tag := range tags {
if strings.Contains(strings.ToLower(tag), searchCriteria) {
return true
}
if strings.Contains(strings.ToLower(group.Name), searchCriteria) {
return true
}
for _, tagID := range group.TagIDs {
if strings.Contains(strings.ToLower(tagsMap[tagID]), searchCriteria) {
return true
}
}
}
@@ -413,11 +413,10 @@ func edgeGroupMatchSearchCriteria(
endpoint *portainer.Endpoint,
edgeGroups []portainer.EdgeGroup,
searchCriteria string,
endpoints []portainer.Endpoint,
endpointGroups []portainer.EndpointGroup,
) bool {
for _, edgeGroup := range edgeGroups {
relatedEndpointIDs := edge.EdgeGroupRelatedEndpoints(&edgeGroup, endpoints, endpointGroups)
relatedEndpointIDs := edge.EdgeGroupRelatedEndpoints(&edgeGroup, []portainer.Endpoint{*endpoint}, endpointGroups)
for _, endpointID := range relatedEndpointIDs {
if endpointID == endpoint.ID {
@@ -448,16 +447,6 @@ func filterEndpointsByTypes(endpoints []portainer.Endpoint, endpointTypes []port
return endpoints[:n]
}
func convertTagIDsToTags(tagsMap map[portainer.TagID]string, tagIDs []portainer.TagID) []string {
tags := make([]string, 0, len(tagIDs))
for _, tagID := range tagIDs {
tags = append(tags, tagsMap[tagID])
}
return tags
}
func filteredEndpointsByTags(endpoints []portainer.Endpoint, tagIDs []portainer.TagID, endpointGroups []portainer.EndpointGroup, partialMatch bool) []portainer.Endpoint {
n := 0
for _, endpoint := range endpoints {

View File

@@ -1,6 +1,7 @@
package endpoints
import (
"strconv"
"testing"
portainer "github.com/portainer/portainer/api"
@@ -148,6 +149,103 @@ func Test_Filter_excludeIDs(t *testing.T) {
runTests(tests, t, handler, environments)
}
func BenchmarkFilterEndpointsBySearchCriteria_PartialMatch(b *testing.B) {
n := 10000
endpointIDs := []portainer.EndpointID{}
endpoints := []portainer.Endpoint{}
for i := range n {
endpoints = append(endpoints, portainer.Endpoint{
ID: portainer.EndpointID(i + 1),
Name: "endpoint-" + strconv.Itoa(i+1),
GroupID: 1,
TagIDs: []portainer.TagID{1},
Type: portainer.EdgeAgentOnDockerEnvironment,
})
endpointIDs = append(endpointIDs, portainer.EndpointID(i+1))
}
endpointGroups := []portainer.EndpointGroup{}
edgeGroups := []portainer.EdgeGroup{}
for i := range 1000 {
edgeGroups = append(edgeGroups, portainer.EdgeGroup{
ID: portainer.EdgeGroupID(i + 1),
Name: "edge-group-" + strconv.Itoa(i+1),
Endpoints: append([]portainer.EndpointID{}, endpointIDs...),
Dynamic: true,
TagIDs: []portainer.TagID{1, 2, 3},
PartialMatch: true,
})
}
tagsMap := map[portainer.TagID]string{}
for i := range 10 {
tagsMap[portainer.TagID(i+1)] = "tag-" + strconv.Itoa(i+1)
}
searchString := "edge-group"
b.ResetTimer()
for range b.N {
e := filterEndpointsBySearchCriteria(endpoints, endpointGroups, edgeGroups, tagsMap, searchString)
if len(e) != n {
b.FailNow()
}
}
}
func BenchmarkFilterEndpointsBySearchCriteria_FullMatch(b *testing.B) {
n := 10000
endpointIDs := []portainer.EndpointID{}
endpoints := []portainer.Endpoint{}
for i := range n {
endpoints = append(endpoints, portainer.Endpoint{
ID: portainer.EndpointID(i + 1),
Name: "endpoint-" + strconv.Itoa(i+1),
GroupID: 1,
TagIDs: []portainer.TagID{1, 2, 3},
Type: portainer.EdgeAgentOnDockerEnvironment,
})
endpointIDs = append(endpointIDs, portainer.EndpointID(i+1))
}
endpointGroups := []portainer.EndpointGroup{}
edgeGroups := []portainer.EdgeGroup{}
for i := range 1000 {
edgeGroups = append(edgeGroups, portainer.EdgeGroup{
ID: portainer.EdgeGroupID(i + 1),
Name: "edge-group-" + strconv.Itoa(i+1),
Endpoints: append([]portainer.EndpointID{}, endpointIDs...),
Dynamic: true,
TagIDs: []portainer.TagID{1},
})
}
tagsMap := map[portainer.TagID]string{}
for i := range 10 {
tagsMap[portainer.TagID(i+1)] = "tag-" + strconv.Itoa(i+1)
}
searchString := "edge-group"
b.ResetTimer()
for range b.N {
e := filterEndpointsBySearchCriteria(endpoints, endpointGroups, edgeGroups, tagsMap, searchString)
if len(e) != n {
b.FailNow()
}
}
}
func runTests(tests []filterTest, t *testing.T, handler *Handler, endpoints []portainer.Endpoint) {
for _, test := range tests {
t.Run(test.title, func(t *testing.T) {

View File

@@ -197,7 +197,7 @@ func (handler *Handler) kubeClientMiddleware(next http.Handler) http.Handler {
return
}
nonAdminNamespaces, err = pcli.GetNonAdminNamespaces(int(user.ID))
nonAdminNamespaces, err = pcli.GetNonAdminNamespaces(int(user.ID), endpoint.Kubernetes.Configuration.RestrictDefaultNamespace)
if err != nil {
httperror.WriteError(w, http.StatusInternalServerError, "an error occurred during the KubeClientMiddleware operation, unable to retrieve non-admin namespaces. Error: ", err)
return

View File

@@ -70,8 +70,8 @@ type TLSInfo struct {
// Existing types
type K8sApplicationResource struct {
CPURequest int64 `json:"CpuRequest"`
CPULimit int64 `json:"CpuLimit"`
MemoryRequest int64 `json:"MemoryRequest"`
MemoryLimit int64 `json:"MemoryLimit"`
CPURequest float64 `json:"CpuRequest"`
CPULimit float64 `json:"CpuLimit"`
MemoryRequest int64 `json:"MemoryRequest"`
MemoryLimit int64 `json:"MemoryLimit"`
}

View File

@@ -77,6 +77,7 @@ func edgeGroupRelatedToEndpoint(edgeGroup *portainer.EdgeGroup, endpoint *portai
return true
}
}
return false
}
@@ -84,12 +85,10 @@ func edgeGroupRelatedToEndpoint(edgeGroup *portainer.EdgeGroup, endpoint *portai
if endpointGroup.TagIDs != nil {
endpointTags = tag.Union(endpointTags, tag.Set(endpointGroup.TagIDs))
}
edgeGroupTags := tag.Set(edgeGroup.TagIDs)
if edgeGroup.PartialMatch {
intersection := tag.Intersection(endpointTags, edgeGroupTags)
return len(intersection) != 0
return tag.PartialMatch(edgeGroup.TagIDs, endpointTags)
}
return tag.FullMatch(edgeGroupTags, endpointTags)
return tag.FullMatch(edgeGroup.TagIDs, endpointTags)
}

View File

@@ -124,13 +124,17 @@ func (kcl *KubeClient) UpdateNamespaceAccessPolicies(accessPolicies map[string]p
}
// GetNonAdminNamespaces retrieves namespaces for a non-admin user, excluding the default namespace if restricted.
func (kcl *KubeClient) GetNonAdminNamespaces(userID int) ([]string, error) {
func (kcl *KubeClient) GetNonAdminNamespaces(userID int, isRestrictDefaultNamespace bool) ([]string, error) {
accessPolicies, err := kcl.GetNamespaceAccessPolicies()
if err != nil {
return nil, fmt.Errorf("an error occurred during the getNonAdminNamespaces operation, unable to get namespace access policies via portainer-config. check if portainer-config configMap exists in the Kubernetes cluster: %w", err)
}
nonAdminNamespaces := []string{defaultNamespace}
nonAdminNamespaces := []string{}
if !isRestrictDefaultNamespace {
nonAdminNamespaces = append(nonAdminNamespaces, defaultNamespace)
}
for namespace, accessPolicy := range accessPolicies {
if hasUserAccessToNamespace(userID, nil, accessPolicy) {
nonAdminNamespaces = append(nonAdminNamespaces, namespace)

View File

@@ -135,8 +135,8 @@ func (kcl *KubeClient) GetApplicationsResource(namespace, node string) (models.K
for _, pod := range pods.Items {
for _, container := range pod.Spec.Containers {
resource.CPURequest += container.Resources.Requests.Cpu().MilliValue()
resource.CPULimit += container.Resources.Limits.Cpu().MilliValue()
resource.CPURequest += float64(container.Resources.Requests.Cpu().MilliValue())
resource.CPULimit += float64(container.Resources.Limits.Cpu().MilliValue())
resource.MemoryRequest += container.Resources.Requests.Memory().Value()
resource.MemoryLimit += container.Resources.Limits.Memory().Value()
}
@@ -356,8 +356,8 @@ func updateApplicationWithService(application models.K8sApplication, services []
func calculateResourceUsage(pod corev1.Pod) models.K8sApplicationResource {
resource := models.K8sApplicationResource{}
for _, container := range pod.Spec.Containers {
resource.CPURequest += container.Resources.Requests.Cpu().MilliValue()
resource.CPULimit += container.Resources.Limits.Cpu().MilliValue()
resource.CPURequest += float64(container.Resources.Requests.Cpu().MilliValue())
resource.CPULimit += float64(container.Resources.Limits.Cpu().MilliValue())
resource.MemoryRequest += container.Resources.Requests.Memory().Value()
resource.MemoryLimit += container.Resources.Limits.Memory().Value()
}

View File

@@ -214,6 +214,7 @@ func (kcl *KubeClient) CombineVolumesWithApplications(volumes *[]models.K8sVolum
hasReplicaSetOwnerReference := containsReplicaSetOwnerReference(pods)
replicaSetItems := make([]appsv1.ReplicaSet, 0)
deploymentItems := make([]appsv1.Deployment, 0)
if hasReplicaSetOwnerReference {
replicaSets, err := kcl.cli.AppsV1().ReplicaSets("").List(context.Background(), metav1.ListOptions{})
if err != nil {
@@ -221,19 +222,48 @@ func (kcl *KubeClient) CombineVolumesWithApplications(volumes *[]models.K8sVolum
return nil, fmt.Errorf("an error occurred during the CombineVolumesWithApplications operation, unable to list replica sets across the cluster. Error: %w", err)
}
replicaSetItems = replicaSets.Items
deployments, err := kcl.cli.AppsV1().Deployments("").List(context.Background(), metav1.ListOptions{})
if err != nil {
log.Error().Err(err).Msg("Failed to list deployments across the cluster")
return nil, fmt.Errorf("an error occurred during the CombineVolumesWithApplications operation, unable to list deployments across the cluster. Error: %w", err)
}
deploymentItems = deployments.Items
}
return kcl.updateVolumesWithOwningApplications(volumes, pods, replicaSetItems)
hasStatefulSetOwnerReference := containsStatefulSetOwnerReference(pods)
statefulSetItems := make([]appsv1.StatefulSet, 0)
if hasStatefulSetOwnerReference {
statefulSets, err := kcl.cli.AppsV1().StatefulSets("").List(context.Background(), metav1.ListOptions{})
if err != nil {
log.Error().Err(err).Msg("Failed to list stateful sets across the cluster")
return nil, fmt.Errorf("an error occurred during the CombineVolumesWithApplications operation, unable to list stateful sets across the cluster. Error: %w", err)
}
statefulSetItems = statefulSets.Items
}
hasDaemonSetOwnerReference := containsDaemonSetOwnerReference(pods)
daemonSetItems := make([]appsv1.DaemonSet, 0)
if hasDaemonSetOwnerReference {
daemonSets, err := kcl.cli.AppsV1().DaemonSets("").List(context.Background(), metav1.ListOptions{})
if err != nil {
log.Error().Err(err).Msg("Failed to list daemon sets across the cluster")
return nil, fmt.Errorf("an error occurred during the CombineVolumesWithApplications operation, unable to list daemon sets across the cluster. Error: %w", err)
}
daemonSetItems = daemonSets.Items
}
return kcl.updateVolumesWithOwningApplications(volumes, pods, deploymentItems, replicaSetItems, statefulSetItems, daemonSetItems)
}
// updateVolumesWithOwningApplications updates the volumes with the applications that use them.
func (kcl *KubeClient) updateVolumesWithOwningApplications(volumes *[]models.K8sVolumeInfo, pods *corev1.PodList, replicaSetItems []appsv1.ReplicaSet) (*[]models.K8sVolumeInfo, error) {
func (kcl *KubeClient) updateVolumesWithOwningApplications(volumes *[]models.K8sVolumeInfo, pods *corev1.PodList, deploymentItems []appsv1.Deployment, replicaSetItems []appsv1.ReplicaSet, statefulSetItems []appsv1.StatefulSet, daemonSetItems []appsv1.DaemonSet) (*[]models.K8sVolumeInfo, error) {
for i, volume := range *volumes {
for _, pod := range pods.Items {
if pod.Spec.Volumes != nil {
for _, podVolume := range pod.Spec.Volumes {
if podVolume.PersistentVolumeClaim != nil && podVolume.PersistentVolumeClaim.ClaimName == volume.PersistentVolumeClaim.Name && pod.Namespace == volume.PersistentVolumeClaim.Namespace {
application, err := kcl.ConvertPodToApplication(pod, replicaSetItems, []appsv1.Deployment{}, []appsv1.StatefulSet{}, []appsv1.DaemonSet{}, []corev1.Service{}, false)
if podVolume.VolumeSource.PersistentVolumeClaim != nil && podVolume.VolumeSource.PersistentVolumeClaim.ClaimName == volume.PersistentVolumeClaim.Name && pod.Namespace == volume.PersistentVolumeClaim.Namespace {
application, err := kcl.ConvertPodToApplication(pod, replicaSetItems, deploymentItems, statefulSetItems, daemonSetItems, []corev1.Service{}, false)
if err != nil {
log.Error().Err(err).Msg("Failed to convert pod to application")
return nil, fmt.Errorf("an error occurred during the CombineServicesWithApplications operation, unable to convert pod to application. Error: %w", err)

View File

@@ -31,19 +31,19 @@ func NewService() *Service {
func (*Service) Authenticate(code string, configuration *portainer.OAuthSettings) (string, error) {
token, err := getOAuthToken(code, configuration)
if err != nil {
log.Debug().Err(err).Msg("failed retrieving oauth token")
log.Error().Err(err).Msg("failed retrieving oauth token")
return "", err
}
idToken, err := getIdToken(token)
if err != nil {
log.Debug().Err(err).Msg("failed parsing id_token")
log.Error().Err(err).Msg("failed parsing id_token")
}
resource, err := getResource(token.AccessToken, configuration)
if err != nil {
log.Debug().Err(err).Msg("failed retrieving resource")
log.Error().Err(err).Msg("failed retrieving resource")
return "", err
}
@@ -52,7 +52,7 @@ func (*Service) Authenticate(code string, configuration *portainer.OAuthSettings
username, err := getUsername(resource, configuration)
if err != nil {
log.Debug().Err(err).Msg("failed retrieving username")
log.Error().Err(err).Msg("failed retrieving username")
return "", err
}

View File

@@ -1599,7 +1599,7 @@ type (
const (
// APIVersion is the version number of the Portainer API
APIVersion = "2.22.0"
APIVersion = "2.22.0-test"
// Edition is what this edition of Portainer is called
Edition = PortainerCE
// ComposeSyntaxMaxVersion is a maximum supported version of the docker compose syntax

View File

@@ -1,64 +1,63 @@
package tag
import portainer "github.com/portainer/portainer/api"
import (
portainer "github.com/portainer/portainer/api"
)
type tagSet map[portainer.TagID]bool
type tagSet map[portainer.TagID]struct{}
// Set converts an array of ids to a set
func Set(tagIDs []portainer.TagID) tagSet {
set := map[portainer.TagID]bool{}
set := map[portainer.TagID]struct{}{}
for _, tagID := range tagIDs {
set[tagID] = true
set[tagID] = struct{}{}
}
return set
}
// Intersection returns a set intersection of the provided sets
func Intersection(sets ...tagSet) tagSet {
intersection := tagSet{}
if len(sets) == 0 {
return intersection
// IntersectionCount returns the element count of the intersection of the sets
func IntersectionCount(setA, setB tagSet) int {
if len(setA) > len(setB) {
setA, setB = setB, setA
}
setA := sets[0]
count := 0
for tag := range setA {
inAll := true
for _, setB := range sets {
if !setB[tag] {
inAll = false
break
}
}
if inAll {
intersection[tag] = true
if _, ok := setB[tag]; ok {
count++
}
}
return intersection
return count
}
// Union returns a set union of provided sets
func Union(sets ...tagSet) tagSet {
union := tagSet{}
for _, set := range sets {
for tag := range set {
union[tag] = true
union[tag] = struct{}{}
}
}
return union
}
// Contains return true if setA contains setB
func Contains(setA tagSet, setB tagSet) bool {
func Contains(setA tagSet, setB []portainer.TagID) bool {
if len(setA) == 0 || len(setB) == 0 {
return false
}
for tag := range setB {
if !setA[tag] {
for _, tag := range setB {
if _, ok := setA[tag]; !ok {
return false
}
}
return true
}
@@ -67,8 +66,8 @@ func Difference(setA tagSet, setB tagSet) tagSet {
set := tagSet{}
for tag := range setA {
if !setB[tag] {
set[tag] = true
if _, ok := setB[tag]; !ok {
set[tag] = struct{}{}
}
}

View File

@@ -1,11 +1,19 @@
package tag
import portainer "github.com/portainer/portainer/api"
// FullMatch returns true if environment tags matches all edge group tags
func FullMatch(edgeGroupTags tagSet, environmentTags tagSet) bool {
func FullMatch(edgeGroupTags []portainer.TagID, environmentTags tagSet) bool {
return Contains(environmentTags, edgeGroupTags)
}
// PartialMatch returns true if environment tags matches at least one edge group tag
func PartialMatch(edgeGroupTags tagSet, environmentTags tagSet) bool {
return len(Intersection(edgeGroupTags, environmentTags)) != 0
func PartialMatch(edgeGroupTags []portainer.TagID, environmentTags tagSet) bool {
for _, tagID := range edgeGroupTags {
if _, ok := environmentTags[tagID]; ok {
return true
}
}
return false
}

View File

@@ -9,49 +9,49 @@ import (
func TestFullMatch(t *testing.T) {
cases := []struct {
name string
edgeGroupTags tagSet
edgeGroupTags []portainer.TagID
environmentTag tagSet
expected bool
}{
{
name: "environment tag partially match edge group tags",
edgeGroupTags: Set([]portainer.TagID{1, 2, 3}),
edgeGroupTags: []portainer.TagID{1, 2, 3},
environmentTag: Set([]portainer.TagID{1, 2}),
expected: false,
},
{
name: "edge group tags equal to environment tags",
edgeGroupTags: Set([]portainer.TagID{1, 2}),
edgeGroupTags: []portainer.TagID{1, 2},
environmentTag: Set([]portainer.TagID{1, 2}),
expected: true,
},
{
name: "environment tags fully match edge group tags",
edgeGroupTags: Set([]portainer.TagID{1, 2}),
edgeGroupTags: []portainer.TagID{1, 2},
environmentTag: Set([]portainer.TagID{1, 2, 3}),
expected: true,
},
{
name: "environment tags do not match edge group tags",
edgeGroupTags: Set([]portainer.TagID{1, 2}),
edgeGroupTags: []portainer.TagID{1, 2},
environmentTag: Set([]portainer.TagID{3, 4}),
expected: false,
},
{
name: "edge group has no tags and environment has tags",
edgeGroupTags: Set([]portainer.TagID{}),
edgeGroupTags: []portainer.TagID{},
environmentTag: Set([]portainer.TagID{1, 2}),
expected: false,
},
{
name: "edge group has tags and environment has no tags",
edgeGroupTags: Set([]portainer.TagID{1, 2}),
edgeGroupTags: []portainer.TagID{1, 2},
environmentTag: Set([]portainer.TagID{}),
expected: false,
},
{
name: "both edge group and environment have no tags",
edgeGroupTags: Set([]portainer.TagID{}),
edgeGroupTags: []portainer.TagID{},
environmentTag: Set([]portainer.TagID{}),
expected: false,
},
@@ -70,55 +70,55 @@ func TestFullMatch(t *testing.T) {
func TestPartialMatch(t *testing.T) {
cases := []struct {
name string
edgeGroupTags tagSet
edgeGroupTags []portainer.TagID
environmentTag tagSet
expected bool
}{
{
name: "environment tags partially match edge group tags 1",
edgeGroupTags: Set([]portainer.TagID{1, 2, 3}),
edgeGroupTags: []portainer.TagID{1, 2, 3},
environmentTag: Set([]portainer.TagID{1, 2}),
expected: true,
},
{
name: "environment tags partially match edge group tags 2",
edgeGroupTags: Set([]portainer.TagID{1, 2, 3}),
edgeGroupTags: []portainer.TagID{1, 2, 3},
environmentTag: Set([]portainer.TagID{1, 4, 5}),
expected: true,
},
{
name: "edge group tags equal to environment tags",
edgeGroupTags: Set([]portainer.TagID{1, 2}),
edgeGroupTags: []portainer.TagID{1, 2},
environmentTag: Set([]portainer.TagID{1, 2}),
expected: true,
},
{
name: "environment tags fully match edge group tags",
edgeGroupTags: Set([]portainer.TagID{1, 2}),
edgeGroupTags: []portainer.TagID{1, 2},
environmentTag: Set([]portainer.TagID{1, 2, 3}),
expected: true,
},
{
name: "environment tags do not match edge group tags",
edgeGroupTags: Set([]portainer.TagID{1, 2}),
edgeGroupTags: []portainer.TagID{1, 2},
environmentTag: Set([]portainer.TagID{3, 4}),
expected: false,
},
{
name: "edge group has no tags and environment has tags",
edgeGroupTags: Set([]portainer.TagID{}),
edgeGroupTags: []portainer.TagID{},
environmentTag: Set([]portainer.TagID{1, 2}),
expected: false,
},
{
name: "edge group has tags and environment has no tags",
edgeGroupTags: Set([]portainer.TagID{1, 2}),
edgeGroupTags: []portainer.TagID{1, 2},
environmentTag: Set([]portainer.TagID{}),
expected: false,
},
{
name: "both edge group and environment have no tags",
edgeGroupTags: Set([]portainer.TagID{}),
edgeGroupTags: []portainer.TagID{},
environmentTag: Set([]portainer.TagID{}),
expected: false,
},

View File

@@ -7,49 +7,49 @@ import (
portainer "github.com/portainer/portainer/api"
)
func TestIntersection(t *testing.T) {
func TestIntersectionCount(t *testing.T) {
cases := []struct {
name string
setA tagSet
setB tagSet
expected tagSet
expected int
}{
{
name: "positive numbers set intersection",
setA: Set([]portainer.TagID{1, 2, 3, 4, 5}),
setB: Set([]portainer.TagID{4, 5, 6, 7}),
expected: Set([]portainer.TagID{4, 5}),
expected: 2,
},
{
name: "empty setA intersection",
setA: Set([]portainer.TagID{1, 2, 3}),
setB: Set([]portainer.TagID{}),
expected: Set([]portainer.TagID{}),
expected: 0,
},
{
name: "empty setB intersection",
setA: Set([]portainer.TagID{}),
setB: Set([]portainer.TagID{1, 2, 3}),
expected: Set([]portainer.TagID{}),
expected: 0,
},
{
name: "no common elements sets intersection",
setA: Set([]portainer.TagID{1, 2, 3}),
setB: Set([]portainer.TagID{4, 5, 6}),
expected: Set([]portainer.TagID{}),
expected: 0,
},
{
name: "equal sets intersection",
setA: Set([]portainer.TagID{1, 2, 3}),
setB: Set([]portainer.TagID{1, 2, 3}),
expected: Set([]portainer.TagID{1, 2, 3}),
expected: 3,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
result := Intersection(tc.setA, tc.setB)
if !reflect.DeepEqual(result, tc.expected) {
result := IntersectionCount(tc.setA, tc.setB)
if result != tc.expected {
t.Errorf("Expected %v, got %v", tc.expected, result)
}
})
@@ -109,49 +109,49 @@ func TestContains(t *testing.T) {
cases := []struct {
name string
setA tagSet
setB tagSet
setB []portainer.TagID
expected bool
}{
{
name: "setA contains setB",
setA: Set([]portainer.TagID{1, 2, 3}),
setB: Set([]portainer.TagID{1, 2}),
setB: []portainer.TagID{1, 2},
expected: true,
},
{
name: "setA equals to setB",
setA: Set([]portainer.TagID{1, 2}),
setB: Set([]portainer.TagID{1, 2}),
setB: []portainer.TagID{1, 2},
expected: true,
},
{
name: "setA contains parts of setB",
setA: Set([]portainer.TagID{1, 2}),
setB: Set([]portainer.TagID{1, 2, 3}),
setB: []portainer.TagID{1, 2, 3},
expected: false,
},
{
name: "setA does not contain setB",
setA: Set([]portainer.TagID{1, 2}),
setB: Set([]portainer.TagID{3, 4}),
setB: []portainer.TagID{3, 4},
expected: false,
},
{
name: "setA is empty and setB is not empty",
setA: Set([]portainer.TagID{}),
setB: Set([]portainer.TagID{1, 2}),
setB: []portainer.TagID{1, 2},
expected: false,
},
{
name: "setA is not empty and setB is empty",
setA: Set([]portainer.TagID{1, 2}),
setB: Set([]portainer.TagID{}),
setB: []portainer.TagID{},
expected: false,
},
{
name: "setA is empty and setB is empty",
setA: Set([]portainer.TagID{}),
setB: Set([]portainer.TagID{}),
setB: []portainer.TagID{},
expected: false,
},
}

View File

@@ -21,7 +21,7 @@ import {
KubernetesApplicationVolumePersistentPayload,
KubernetesApplicationVolumeSecretPayload,
} from 'Kubernetes/models/application/payloads';
import KubernetesVolumeHelper from 'Kubernetes/helpers/volumeHelper';
import { generatedApplicationConfigVolumeName } from '@/react/kubernetes/volumes/utils';
import { HelmApplication } from 'Kubernetes/models/application/models';
import { KubernetesApplicationDeploymentTypes, KubernetesApplicationTypes } from 'Kubernetes/models/application/models/appConstants';
import { KubernetesPodAffinity, KubernetesPodNodeAffinityNodeSelectorRequirementOperators } from 'Kubernetes/pod/models';
@@ -239,7 +239,7 @@ class KubernetesApplicationHelper {
const volKeys = _.filter(config.overridenKeys, (item) => item.type === 'FILESYSTEM');
const groupedVolKeys = _.groupBy(volKeys, 'path');
_.forEach(groupedVolKeys, (items, path) => {
const volumeName = KubernetesVolumeHelper.generatedApplicationConfigVolumeName(app.Name);
const volumeName = generatedApplicationConfigVolumeName(app.Name);
const configurationName = config.selectedConfiguration.metadata.name;
const itemsMap = _.map(items, (item) => {
const entry = new KubernetesApplicationVolumeEntryPayload();

View File

@@ -1,5 +1,4 @@
import _ from 'lodash-es';
import uuidv4 from 'uuid/v4';
import { KubernetesApplicationTypes } from 'Kubernetes/models/application/models/appConstants';
class KubernetesVolumeHelper {
@@ -20,18 +19,6 @@ class KubernetesVolumeHelper {
);
});
}
static isUsed(item) {
return item.Applications.length !== 0;
}
static generatedApplicationConfigVolumeName(name) {
return 'config-' + name + '-' + uuidv4();
}
static isExternalVolume(volume) {
return !volume.PersistentVolumeClaim.ApplicationOwner;
}
}
export default KubernetesVolumeHelper;

View File

@@ -29,6 +29,7 @@ import { confirm, confirmUpdate, confirmWebEditorDiscard } from '@@/modals/confi
import { buildConfirmButton } from '@@/modals/utils';
import { ModalType } from '@@/modals';
import { KUBE_STACK_NAME_VALIDATION_REGEX } from '@/react/kubernetes/DeployView/StackName/constants';
import { isVolumeUsed } from '@/react/kubernetes/volumes/utils';
class KubernetesCreateApplicationController {
/* #region CONSTRUCTOR */
@@ -785,7 +786,7 @@ class KubernetesCreateApplicationController {
});
this.volumes = volumes;
const filteredVolumes = _.filter(this.volumes, (volume) => {
const isUnused = !KubernetesVolumeHelper.isUsed(volume);
const isUnused = !isVolumeUsed(volume);
const isRWX = volume.PersistentVolumeClaim.storageClass && _.includes(volume.PersistentVolumeClaim.storageClass.AccessModes, 'RWX');
return isUnused || isRWX;
});

View File

@@ -310,7 +310,9 @@ class KubernetesNodeController {
}
getNodeUsage() {
return this.$async(this.getNodeUsageAsync);
if (this.hasResourceUsageAccess()) {
return this.$async(this.getNodeUsageAsync);
}
}
hasEventWarnings() {
@@ -361,9 +363,7 @@ class KubernetesNodeController {
useServerMetrics: this.endpoint.Kubernetes.Configuration.UseServerMetrics,
};
await this.getNodes();
await this.getEvents();
await this.getEndpoints();
await Promise.allSettled([this.getNodes(), this.getEvents(), this.getEndpoints(), this.getNodeUsage()]);
this.availableEffects = _.values(KubernetesNodeTaintEffects);
this.formValues = KubernetesNodeConverter.nodeToFormValues(this.node);

View File

@@ -6,6 +6,7 @@ import KubernetesEventHelper from 'Kubernetes/helpers/eventHelper';
import { KubernetesStorageClassAccessPolicies } from 'Kubernetes/models/storage-class/models';
import KubernetesNamespaceHelper from 'Kubernetes/helpers/namespaceHelper';
import { confirmRedeploy } from '@/react/kubernetes/volumes/ItemView/ConfirmRedeployModal';
import { isVolumeUsed, isVolumeExternal } from '@/react/kubernetes/volumes/utils';
class KubernetesVolumeController {
/* @ngInject */
@@ -49,7 +50,7 @@ class KubernetesVolumeController {
}
isExternalVolume() {
return KubernetesVolumeHelper.isExternalVolume(this.volume);
return isVolumeExternal(this.volume);
}
isSystemNamespace() {
@@ -57,7 +58,7 @@ class KubernetesVolumeController {
}
isUsed() {
return KubernetesVolumeHelper.isUsed(this.volume);
return isVolumeUsed(this.volume);
}
onChangeSize() {
@@ -102,7 +103,7 @@ class KubernetesVolumeController {
}
updateVolume() {
if (KubernetesVolumeHelper.isUsed(this.volume)) {
if (isVolumeUsed(this.volume)) {
confirmRedeploy().then((redeploy) => {
return this.$async(this.updateVolumeAsync, redeploy);
});

View File

@@ -139,7 +139,7 @@ function ErrorCell({
</div>
<div
className={clsx('overflow-hidden whitespace-normal', {
'h-[1.5em]': isExpanded,
'h-[1.5em]': !isExpanded,
})}
>
{value}

View File

@@ -1,7 +1,6 @@
import { Database } from 'lucide-react';
import { Authorized, useAuthorizations } from '@/react/hooks/useUser';
import KubernetesVolumeHelper from '@/kubernetes/helpers/volumeHelper';
import { useEnvironmentId } from '@/react/hooks/useEnvironmentId';
import { refreshableSettings } from '@@/datatables/types';
@@ -20,6 +19,7 @@ import { useNamespacesQuery } from '../../namespaces/queries/useNamespacesQuery'
import { useAllVolumesQuery } from '../queries/useVolumesQuery';
import { isSystemNamespace } from '../../namespaces/queries/useIsSystemNamespace';
import { useDeleteVolumes } from '../queries/useDeleteVolumes';
import { isVolumeUsed } from '../utils';
import { columns } from './columns';
@@ -68,7 +68,7 @@ export function VolumesDatatable() {
disableSelect={!hasWriteAuth}
isRowSelectable={({ original: volume }) =>
!isSystemNamespace(volume.ResourcePool.Namespace.Name, namespaces) &&
!KubernetesVolumeHelper.isUsed(volume)
!isVolumeUsed(volume)
}
renderTableActions={(selectedItems) => (
<Authorized authorizations="K8sVolumesW">

View File

@@ -1,6 +1,5 @@
import { CellContext } from '@tanstack/react-table';
import KubernetesVolumeHelper from '@/kubernetes/helpers/volumeHelper';
import { useEnvironmentId } from '@/react/hooks/useEnvironmentId';
import { Link } from '@@/Link';
@@ -9,6 +8,7 @@ import { ExternalBadge } from '@@/Badge/ExternalBadge';
import { UnusedBadge } from '@@/Badge/UnusedBadge';
import { useNamespacesQuery } from '../../namespaces/queries/useNamespacesQuery';
import { isVolumeExternal, isVolumeUsed } from '../utils';
import { VolumeViewModel } from './types';
import { helper } from './columns.helper';
@@ -44,8 +44,8 @@ export function NameCell({
<SystemBadge />
) : (
<>
{KubernetesVolumeHelper.isExternalVolume(item) && <ExternalBadge />}
{!KubernetesVolumeHelper.isUsed(item) && <UnusedBadge />}
{isVolumeExternal(item) && <ExternalBadge />}
{!isVolumeUsed(item) && <UnusedBadge />}
</>
)}
</div>

View File

@@ -0,0 +1,15 @@
import uuidv4 from 'uuid/v4';
import { VolumeViewModel } from './ListView/types';
export function isVolumeUsed(volume: VolumeViewModel) {
return volume.Applications.length !== 0;
}
export function isVolumeExternal(volume: VolumeViewModel) {
return !volume.PersistentVolumeClaim.ApplicationOwner;
}
export function generatedApplicationConfigVolumeName(applicationName: string) {
return `config-${applicationName}-${uuidv4()}`;
}