fix(app): fix pull rate limit checker

This commit is contained in:
LP B
2021-05-24 18:07:12 +02:00
parent 918e46be0e
commit 7c8c251021
26 changed files with 175 additions and 79 deletions

View File

@@ -22,7 +22,7 @@ type dockerhubStatusResponse struct {
Limit int `json:"limit"`
}
// GET request on /api/endpoints/{id}/dockerhub/status
// GET request on /api/endpoints/{id}/dockerhub/{registryId}
func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
@@ -40,13 +40,30 @@ func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.R
return &httperror.HandlerError{http.StatusBadRequest, "Invalid environment type", errors.New("Invalid environment type")}
}
dockerhub, err := handler.DataStore.DockerHub().DockerHub()
registryID, err := request.RetrieveNumericRouteVariableValue(r, "registryId")
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub details from the database", err}
return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry identifier route variable", err}
}
var registry *portainer.Registry
if registryID == 0 {
registry = &portainer.Registry{}
} else {
registry, err = handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
if err == bolterrors.ErrObjectNotFound {
return &httperror.HandlerError{http.StatusNotFound, "Unable to find a registry with the specified identifier inside the database", err}
} else if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a registry with the specified identifier inside the database", err}
}
if registry.Type != portainer.DockerHubRegistry {
return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry type", errors.New("Invalid registry type")}
}
}
httpClient := client.NewHTTPClient()
token, err := getDockerHubToken(httpClient, dockerhub)
token, err := getDockerHubToken(httpClient, registry)
if err != nil {
return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub token from DockerHub", err}
}
@@ -59,7 +76,7 @@ func (handler *Handler) endpointDockerhubStatus(w http.ResponseWriter, r *http.R
return response.JSON(w, resp)
}
func getDockerHubToken(httpClient *client.HTTPClient, dockerhub *portainer.DockerHub) (string, error) {
func getDockerHubToken(httpClient *client.HTTPClient, registry *portainer.Registry) (string, error) {
type dockerhubTokenResponse struct {
Token string `json:"token"`
}
@@ -71,8 +88,8 @@ func getDockerHubToken(httpClient *client.HTTPClient, dockerhub *portainer.Docke
return "", err
}
if dockerhub.Authentication {
req.SetBasicAuth(dockerhub.Username, dockerhub.Password)
if registry.Authentication {
req.SetBasicAuth(registry.Username, registry.Password)
}
resp, err := httpClient.Do(req)

View File

@@ -87,7 +87,7 @@ func (handler *Handler) isNamespaceAuthorized(endpoint *portainer.Endpoint, name
return false, nil
}
return !security.AuthorizedAccess(userId, memberships, namespacePolicy.UserAccessPolicies, namespacePolicy.TeamAccessPolicies), nil
return security.AuthorizedAccess(userId, memberships, namespacePolicy.UserAccessPolicies, namespacePolicy.TeamAccessPolicies), nil
}
func filterRegistriesByNamespace(registries []portainer.Registry, endpointId portainer.EndpointID, namespace string) []portainer.Registry {

View File

@@ -53,7 +53,7 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointUpdate))).Methods(http.MethodPut)
h.Handle("/endpoints/{id}",
bouncer.AdminAccess(httperror.LoggerHandler(h.endpointDelete))).Methods(http.MethodDelete)
h.Handle("/endpoints/{id}/dockerhub",
h.Handle("/endpoints/{id}/dockerhub/{registryId}",
bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointDockerhubStatus))).Methods(http.MethodGet)
h.Handle("/endpoints/{id}/extensions",
bouncer.RestrictedAccess(httperror.LoggerHandler(h.endpointExtensionAdd))).Methods(http.MethodPost)

View File

@@ -13,6 +13,7 @@ import (
"strings"
"github.com/docker/docker/client"
"github.com/portainer/libhttp/request"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/docker"
"github.com/portainer/portainer/api/http/proxy/factory/utils"
@@ -166,12 +167,21 @@ func (transport *Transport) proxyAgentRequest(r *http.Request) (*http.Response,
// volume browser request
return transport.restrictedResourceOperation(r, resourceID, portainer.VolumeResourceControl, true)
case strings.HasPrefix(requestPath, "/dockerhub"):
dockerhub, err := transport.dataStore.DockerHub().DockerHub()
registryID, err := request.RetrieveNumericRouteVariableValue(r, "registryId")
if err != nil {
return nil, err
}
newBody, err := json.Marshal(dockerhub)
registry, err := transport.dataStore.Registry().Registry(portainer.RegistryID(registryID))
if err != nil {
return nil, err
}
if registry.Type != portainer.DockerHubRegistry {
return nil, errors.New("Invalid registry type")
}
newBody, err := json.Marshal(registry)
if err != nil {
return nil, err
}

View File

@@ -3,6 +3,7 @@ package kubernetes
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
@@ -10,6 +11,7 @@ import (
"regexp"
"strings"
"github.com/portainer/libhttp/request"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/kubernetes/cli"
@@ -135,12 +137,21 @@ func decorateAgentRequest(r *http.Request, dataStore portainer.DataStore) error
}
func decorateAgentDockerHubRequest(r *http.Request, dataStore portainer.DataStore) error {
dockerhub, err := dataStore.DockerHub().DockerHub()
registryID, err := request.RetrieveNumericRouteVariableValue(r, "registryId")
if err != nil {
return err
}
newBody, err := json.Marshal(dockerhub)
registry, err := dataStore.Registry().Registry(portainer.RegistryID(registryID))
if err != nil {
return err
}
if registry.Type != portainer.DockerHubRegistry {
return errors.New("invalid registry type")
}
newBody, err := json.Marshal(registry)
if err != nil {
return err
}

View File

@@ -17,6 +17,28 @@ type (
namespaceAccessPolicies map[string]accessPolicies
)
// GetNamespaceAccessPolicies gets the namespace access policies
// from config maps in the portainer namespace
func (kcl *KubeClient) GetNamespaceAccessPolicies() (
map[string]portainer.K8sNamespaceAccessPolicy, error,
) {
configMap, err := kcl.cli.CoreV1().ConfigMaps(portainerNamespace).Get(portainerConfigMapName, metav1.GetOptions{})
if k8serrors.IsNotFound(err) {
return nil, nil
} else if err != nil {
return nil, err
}
accessData := configMap.Data[portainerConfigMapAccessPoliciesKey]
var policies map[string]portainer.K8sNamespaceAccessPolicy
err = json.Unmarshal([]byte(accessData), &policies)
if err != nil {
return nil, err
}
return policies, nil
}
func (kcl *KubeClient) setupNamespaceAccesses(userID int, teamIDs []int, serviceAccountName string) error {
configMap, err := kcl.cli.CoreV1().ConfigMaps(portainerNamespace).Get(portainerConfigMapName, metav1.GetOptions{})
if k8serrors.IsNotFound(err) {

View File

@@ -390,6 +390,11 @@ type (
// JobType represents a job type
JobType int
K8sNamespaceAccessPolicy struct {
UserAccessPolicies UserAccessPolicies `json:"UserAccessPolicies"`
TeamAccessPolicies TeamAccessPolicies `json:"TeamAccessPolicies"`
}
// KubernetesData contains all the Kubernetes related endpoint information
KubernetesData struct {
Snapshots []KubernetesSnapshot `json:"Snapshots"`
@@ -1159,6 +1164,7 @@ type (
SetupUserServiceAccount(userID int, teamIDs []int) error
GetServiceAccountBearerToken(userID int) (string, error)
StartExecProcess(namespace, podName, containerName string, command []string, stdin io.Reader, stdout io.Writer) error
GetNamespaceAccessPolicies() (map[string]K8sNamespaceAccessPolicy, error)
DeleteRegistrySecret(registry *Registry, namespace string) error
CreateRegistrySecret(registry *Registry, namespace string) error
IsRegistrySecret(namespace, secretName string) (bool, error)

View File

@@ -4,10 +4,10 @@ angular.module('portainer.agent').factory('AgentDockerhub', AgentDockerhub);
function AgentDockerhub($resource, API_ENDPOINT_ENDPOINTS) {
return $resource(
`${API_ENDPOINT_ENDPOINTS}/:endpointId/:endpointType/v2/dockerhub`,
`${API_ENDPOINT_ENDPOINTS}/:endpointId/:endpointType/v2/dockerhub/:registryId`,
{},
{
limits: { method: 'GET' },
limits: { method: 'GET', params: { registryId: '@registryId' } },
}
);
}

View File

@@ -1,24 +1,25 @@
import EndpointHelper from 'Portainer/helpers/endpointHelper';
export default class porImageRegistryContainerController {
/* @ngInject */
constructor(EndpointHelper, DockerHubService, Notifications) {
this.EndpointHelper = EndpointHelper;
constructor(DockerHubService, Notifications) {
this.DockerHubService = DockerHubService;
this.Notifications = Notifications;
this.pullRateLimits = null;
}
$onChanges({ isDockerHubRegistry }) {
if (isDockerHubRegistry && isDockerHubRegistry.currentValue) {
$onChanges({ registry }) {
if (registry && registry.currentValue && this.isDockerHubRegistry) {
this.fetchRateLimits();
}
}
async fetchRateLimits() {
this.pullRateLimits = null;
if (this.EndpointHelper.isAgentEndpoint(this.endpoint) || this.EndpointHelper.isLocalEndpoint(this.endpoint)) {
if (EndpointHelper.isAgentEndpoint(this.endpoint) || EndpointHelper.isLocalEndpoint(this.endpoint)) {
try {
this.pullRateLimits = await this.DockerHubService.checkRateLimits(this.endpoint);
this.pullRateLimits = await this.DockerHubService.checkRateLimits(this.endpoint, this.registry.Id);
this.setValidity(this.pullRateLimits.remaining >= 0);
} catch (e) {
// eslint-disable-next-line no-console

View File

@@ -5,6 +5,7 @@ import controller from './por-image-registry-rate-limits.controller';
angular.module('portainer.docker').component('porImageRegistryRateLimits', {
bindings: {
endpoint: '<',
registry: '<',
setValidity: '<',
isAdmin: '<',
isDockerHubRegistry: '<',

View File

@@ -52,7 +52,7 @@ class porImageRegistryController {
}
isDockerHubRegistry() {
return this.model.UseRegistry && this.model.Registry.Name === 'DockerHub';
return this.model.UseRegistry && (this.model.Registry.Type === RegistryTypes.DOCKERHUB || this.model.Registry.Type === RegistryTypes.ANONYMOUS);
}
async onRegistryChange() {

View File

@@ -88,6 +88,7 @@
ng-show="$ctrl.checkRateLimits"
is-docker-hub-registry="$ctrl.isDockerHubRegistry()"
endpoint="$ctrl.endpoint"
registry="$ctrl.model.Registry"
set-validity="$ctrl.setValidity"
is-authenticated="$ctrl.model.Registry.Authentication"
is-admin="$ctrl.isAdmin"

View File

@@ -3,7 +3,6 @@ angular.module('portainer.docker').component('porImageRegistry', {
controller: 'porImageRegistryController',
bindings: {
model: '=', // must be of type PorImageRegistryModel
pullWarning: '<',
autoComplete: '<',
labelClass: '@',
inputClass: '@',

View File

@@ -40,7 +40,6 @@
<!-- image-and-registry -->
<por-image-registry
model="formValues.RegistryModel"
pull-warning="formValues.alwaysPull"
ng-if="formValues.RegistryModel.Registry"
auto-complete="true"
label-class="col-sm-1"

View File

@@ -17,7 +17,6 @@
<por-image-registry
model="formValues.RegistryModel"
auto-complete="true"
pull-warning="true"
label-class="col-sm-1"
input-class="col-sm-11"
endpoint-id="endpoint.Id"

View File

@@ -1,6 +1,6 @@
export class EditEdgeGroupController {
/* @ngInject */
constructor(EdgeGroupService, GroupService, TagService, Notifications, $state, $async, EndpointService, EndpointHelper) {
constructor(EdgeGroupService, GroupService, TagService, Notifications, $state, $async, EndpointService) {
this.EdgeGroupService = EdgeGroupService;
this.GroupService = GroupService;
this.TagService = TagService;
@@ -8,7 +8,6 @@ export class EditEdgeGroupController {
this.$state = $state;
this.$async = $async;
this.EndpointService = EndpointService;
this.EndpointHelper = EndpointHelper;
this.state = {
actionInProgress: false,

View File

@@ -95,6 +95,10 @@
input-class="col-sm-11"
endpoint-id="ctrl.endpoint.Id"
namespace="ctrl.formValues.ResourcePool.Namespace.Name"
endpoint="ctrl.endpoint"
is-admin="ctrl.isAdmin"
check-rate-limits="true"
set-validity="ctrl.setPullImageValidity"
></por-image-registry>
</div>
</div>
@@ -105,14 +109,6 @@
</div>
</div>
</div>
<por-image-registry-rate-limits
is-docker-hub-registry="true"
endpoint="ctrl.endpoint"
is-authenticated="ctrl.state.isDockerAuthenticated"
is-admin="ctrl.isAdmin"
set-validity="ctrl.setPullImageValidity"
>
</por-image-registry-rate-limits>
<!-- #endregion -->
<div class="col-sm-12 form-section-title">

View File

@@ -39,7 +39,6 @@ class KubernetesCreateApplicationController {
$state,
Notifications,
Authentication,
DockerHubService,
ModalService,
KubernetesResourcePoolService,
KubernetesApplicationService,
@@ -56,7 +55,6 @@ class KubernetesCreateApplicationController {
this.$state = $state;
this.Notifications = Notifications;
this.Authentication = Authentication;
this.DockerHubService = DockerHubService;
this.ModalService = ModalService;
this.KubernetesResourcePoolService = KubernetesResourcePoolService;
this.KubernetesApplicationService = KubernetesApplicationService;
@@ -999,9 +997,6 @@ class KubernetesCreateApplicationController {
this.formValues.OriginalIngressClasses = angular.copy(this.ingresses);
}
this.updateSliders();
const dockerHub = await this.DockerHubService.dockerhub();
this.state.isDockerAuthenticated = dockerHub.Authentication;
} catch (err) {
this.Notifications.error('Failure', err, 'Unable to load view data');
} finally {

View File

@@ -59,7 +59,7 @@
ng-class="{ active: item.Checked }"
>
<td>
<span class="md-checkbox" ng-if="$ctrl.isAdmin && !ĉtrl.endpointType">
<span class="md-checkbox" ng-if="$ctrl.isAdmin && !ctrl.endpointType">
<input id="select_{{ $index }}" type="checkbox" ng-model="item.Checked" ng-click="$ctrl.selectItem(item, $event)" ng-disabled="!$ctrl.allowSelection(item)" />
<label for="select_{{ $index }}"></label>
</span>

View File

@@ -1,36 +1,33 @@
import _ from 'lodash-es';
import { PortainerEndpointTypes } from 'Portainer/models/endpoint/models';
angular.module('portainer.app').factory('EndpointHelper', [
function EndpointHelperFactory() {
'use strict';
var helper = {};
function findAssociatedGroup(endpoint, groups) {
return _.find(groups, function (group) {
return group.Id === endpoint.GroupId;
});
}
function findAssociatedGroup(endpoint, groups) {
return _.find(groups, function (group) {
return group.Id === endpoint.GroupId;
});
}
export default class EndpointHelper {
static isLocalEndpoint(endpoint) {
return endpoint.URL.includes('unix://') || endpoint.URL.includes('npipe://') || endpoint.Type === PortainerEndpointTypes.KubernetesLocalEnvironment;
}
helper.isLocalEndpoint = isLocalEndpoint;
function isLocalEndpoint(endpoint) {
return endpoint.URL.includes('unix://') || endpoint.URL.includes('npipe://') || endpoint.Type === 5;
}
static isAgentEndpoint(endpoint) {
return [
PortainerEndpointTypes.AgentOnDockerEnvironment,
PortainerEndpointTypes.EdgeAgentOnDockerEnvironment,
PortainerEndpointTypes.AgentOnKubernetesEnvironment,
PortainerEndpointTypes.EdgeAgentOnKubernetesEnvironment,
].includes(endpoint.Type);
}
helper.isAgentEndpoint = isAgentEndpoint;
function isAgentEndpoint(endpoint) {
return [2, 4, 6, 7].includes(endpoint.Type);
}
helper.mapGroupNameToEndpoint = function (endpoints, groups) {
for (var i = 0; i < endpoints.length; i++) {
var endpoint = endpoints[i];
var group = findAssociatedGroup(endpoint, groups);
if (group) {
endpoint.GroupName = group.Name;
}
static mapGroupNameToEndpoint(endpoints, groups) {
for (var i = 0; i < endpoints.length; i++) {
var endpoint = endpoints[i];
var group = findAssociatedGroup(endpoint, groups);
if (group) {
endpoint.GroupName = group.Name;
}
};
return helper;
},
]);
}
}
}

View File

@@ -1,6 +1,7 @@
import { RegistryTypes } from './registryTypes';
export function DockerHubViewModel() {
this.Id = 0;
this.Type = RegistryTypes.ANONYMOUS;
this.Name = 'DockerHub (anonymous)';
this.URL = 'docker.io';

View File

@@ -22,9 +22,22 @@ angular.module('portainer.app').factory('Endpoints', [
snapshot: { method: 'POST', params: { id: '@id', action: 'snapshot' } },
status: { method: 'GET', params: { id: '@id', action: 'status' } },
updateSecuritySettings: { method: 'PUT', params: { id: '@id', action: 'settings' } },
dockerhubLimits: { method: 'GET', params: { id: '@id', action: 'dockerhub' } },
registries: { url: `${API_ENDPOINT_ENDPOINTS}/:id/registries`, method: 'GET', params: { id: '@id', namespace: '@namespace' }, isArray: true },
updateRegistryAccess: { url: `${API_ENDPOINT_ENDPOINTS}/:id/registries/:registryId`, method: 'PUT', params: { id: '@id', registryId: '@registryId' } },
dockerhubLimits: {
method: 'GET',
url: `${API_ENDPOINT_ENDPOINTS}/:id/dockerhub/:registryId`,
params: { id: '@id', registryId: '@registryId' },
},
registries: {
method: 'GET',
url: `${API_ENDPOINT_ENDPOINTS}/:id/registries`,
params: { id: '@id', namespace: '@namespace' },
isArray: true,
},
updateRegistryAccess: {
method: 'PUT',
url: `${API_ENDPOINT_ENDPOINTS}/:id/registries/:registryId`,
params: { id: '@id', registryId: '@registryId' },
},
}
);
},

View File

@@ -0,0 +1,27 @@
import EndpointHelper from 'Portainer/helpers/endpointHelper';
import { PortainerEndpointTypes } from 'Portainer/models/endpoint/models';
angular.module('portainer.app').factory('DockerHubService', DockerHubService);
/* @ngInject */
function DockerHubService(Endpoints, AgentDockerhub) {
return {
checkRateLimits,
};
function checkRateLimits(endpoint, registryId) {
if (EndpointHelper.isLocalEndpoint(endpoint)) {
return Endpoints.dockerhubLimits({ id: endpoint.Id, registryId }).$promise;
}
switch (endpoint.Type) {
case PortainerEndpointTypes.AgentOnDockerEnvironment:
case PortainerEndpointTypes.EdgeAgentOnDockerEnvironment:
return AgentDockerhub.limits({ endpointId: endpoint.Id, endpointType: 'docker', registryId }).$promise;
case PortainerEndpointTypes.AgentOnKubernetesEnvironment:
case PortainerEndpointTypes.EdgeAgentOnKubernetesEnvironment:
return AgentDockerhub.limits({ endpointId: endpoint.Id, endpointType: 'kubernetes', registryId }).$promise;
}
}
}

View File

@@ -41,10 +41,10 @@ angular.module('portainer.app').factory('RegistryService', [
return deferred.promise;
}
function registry(id) {
function registry(id, endpointId) {
var deferred = $q.defer();
Registries.get({ id: id })
Registries.get({ id, endpointId })
.$promise.then(function success(data) {
var registry = new RegistryViewModel(data);
deferred.resolve(registry);
@@ -58,7 +58,7 @@ angular.module('portainer.app').factory('RegistryService', [
function encodedCredentials(registry) {
var credentials = {
serveraddress: registry.URL,
registryId: registry.Id,
};
return btoa(JSON.stringify(credentials));
}

View File

@@ -1,8 +1,9 @@
import angular from 'angular';
import EndpointHelper from 'Portainer/helpers/endpointHelper';
angular.module('portainer.app').controller('EndpointsController', EndpointsController);
function EndpointsController($q, $scope, $state, $async, EndpointService, GroupService, EndpointHelper, Notifications) {
function EndpointsController($q, $scope, $state, $async, EndpointService, GroupService, Notifications) {
$scope.removeAction = removeAction;
function removeAction(endpoints) {

View File

@@ -1,3 +1,5 @@
import EndpointHelper from 'Portainer/helpers/endpointHelper';
angular
.module('portainer.app')
.controller('HomeController', function (
@@ -7,7 +9,6 @@ angular
TagService,
Authentication,
EndpointService,
EndpointHelper,
GroupService,
Notifications,
EndpointProvider,