add missing import

* chore(build-system): update dev-toolkit

* chore(build-system): update dev-toolkit

* chore(build-system): update dev-toolkit Dockerfile

* chore(build-system): update gruntfile

* chore(build-system): gruntfile update

* chore(build-system): better support for private git repositories

* Update toolkit.Dockerfile

* merge develop into toolkit-update

* merge develop into toolkit-update

api/exec/kubernetes_deploy.go

@@ -94,7 +94,9 @@ func (deployer *KubernetesDeployer) Deploy(request *http.Request, endpoint *port
 		args = append(args, "--server", endpoint.URL)
 		args = append(args, "--insecure-skip-tls-verify")
 		args = append(args, "--token", token)
-		args = append(args, "--namespace", namespace)
+		if (namespace != "") {
+			args = append(args, "--namespace", namespace)
+		}
 		args = append(args, "apply", "-f", "-")
 
 		var stderr bytes.Buffer

api/http/handler/stacks/create_kubernetes_stack.go

@@ -15,6 +15,7 @@ import (
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/filesystem"
+	"github.com/portainer/portainer/api/http/client"
 )
 
 const defaultReferenceName = "refs/heads/master"
@@ -36,6 +37,12 @@ type kubernetesGitDeploymentPayload struct {
 	FilePathInRepository     string
 }
 
+type kubernetesManifestURLDeploymentPayload struct {
+	ComposeFormat bool
+	ManifestURL string
+}
+
+
 func (payload *kubernetesStringDeploymentPayload) Validate(r *http.Request) error {
 	if govalidator.IsNull(payload.StackFileContent) {
 		return errors.New("Invalid stack file content")
@@ -65,6 +72,13 @@ func (payload *kubernetesGitDeploymentPayload) Validate(r *http.Request) error {
 	return nil
 }
 
+func (payload *kubernetesManifestURLDeploymentPayload) Validate(r *http.Request) error {
+	if govalidator.IsNull(payload.ManifestURL) || !govalidator.IsURL(payload.ManifestURL) {
+		return errors.New("Invalid manifest URL")
+	}
+	return nil
+}
+
 type createKubernetesStackResponse struct {
 	Output string `json:"Output"`
 }
@@ -155,6 +169,55 @@ func (handler *Handler) createKubernetesStackFromGitRepository(w http.ResponseWr
 	return response.JSON(w, resp)
 }
 
+func (handler *Handler) createKubernetesStackFromManifestURL(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint) *httperror.HandlerError {
+	var payload kubernetesManifestURLDeploymentPayload
+	if err := request.DecodeAndValidateJSONPayload(r, &payload); err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err}
+	}
+
+	stackID := handler.DataStore.Stack().GetNextIdentifier()
+	stack := &portainer.Stack{
+		ID:           portainer.StackID(stackID),
+		Type:         portainer.KubernetesStack,
+		EndpointID:   endpoint.ID,
+		EntryPoint:   filesystem.ManifestFileDefaultName,
+		Status:       portainer.StackStatusActive,
+		CreationDate: time.Now().Unix(),
+	}
+
+	var manifestContent []byte
+	manifestContent, err := client.Get(payload.ManifestURL, 30)
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve manifest from URL", err}
+	}
+	
+	stackFolder := strconv.Itoa(int(stack.ID))
+	projectPath, err := handler.FileService.StoreStackFileFromBytes(stackFolder, stack.EntryPoint, manifestContent)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist Kubernetes manifest file on disk", Err: err}
+	}
+	stack.ProjectPath = projectPath
+
+	doCleanUp := true
+	defer handler.cleanUp(stack, &doCleanUp)
+
+	output, err := handler.deployKubernetesStack(r, endpoint, string(manifestContent), payload.ComposeFormat, "")
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to deploy Kubernetes stack", Err: err}
+	}
+
+	err = handler.DataStore.Stack().CreateStack(stack)
+	if err != nil {
+		return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the Kubernetes stack inside the database", Err: err}
+	}
+
+	resp := &createKubernetesStackResponse{
+		Output: output,
+	}
+
+	return response.JSON(w, resp)
+}
+
 func (handler *Handler) deployKubernetesStack(request *http.Request, endpoint *portainer.Endpoint, stackConfig string, composeFormat bool, namespace string) (string, error) {
 	handler.stackCreationMutex.Lock()
 	defer handler.stackCreationMutex.Unlock()

api/http/handler/stacks/stack_create.go

@@ -149,6 +149,8 @@ func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Req
 		return handler.createKubernetesStackFromFileContent(w, r, endpoint)
 	case "repository":
 		return handler.createKubernetesStackFromGitRepository(w, r, endpoint)
+	case "url":
+		return handler.createKubernetesStackFromManifestURL(w, r, endpoint)
 	}
 	return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid value for query parameter: method. Value must be one of: string or repository", Err: errors.New(request.ErrInvalidQueryParameter)}
 }

app/kubernetes/models/deploy.js

@@ -6,9 +6,11 @@ export const KubernetesDeployManifestTypes = Object.freeze({
 export const KubernetesDeployBuildMethods = Object.freeze({
   GIT: 1,
   WEB_EDITOR: 2,
+  URL: 3
 });
 
 export const KubernetesDeployRequestMethods = Object.freeze({
   REPOSITORY: 'repository',
   STRING: 'string',
+  URL: 'url'
 });

app/kubernetes/views/deploy/deploy.html

@@ -16,7 +16,7 @@
               <uib-tab-heading> <i class="fa fa-code space-right" aria-hidden="true"></i> Deploy </uib-tab-heading>
 
               <form class="form-horizontal" style="margin-top: 20px;">
-                <div class="form-group">
+                <div class="form-group" ng-show="ctrl.state.BuildMethod !== ctrl.BuildMethods.URL">
                   <label for="target_node" class="col-lg-1 col-sm-2 control-label text-left">Namespace</label>
                   <div class="col-lg-11 col-sm-10">
                     <select class="form-control" ng-model="ctrl.formValues.Namespace" ng-options="namespace.Name as namespace.Name for namespace in ctrl.namespaces"></select>
@@ -101,6 +101,33 @@
                 </web-editor-form>
 
                 <!-- !editor -->
+
+                <!-- url -->
+                <div ng-show="ctrl.state.BuildMethod === ctrl.BuildMethods.URL">
+                  <div class="col-sm-12 form-section-title">
+                    URL
+                  </div>
+                  <div class="form-group">
+                    <span class="col-sm-12 text-muted small">
+                      Indicate the URL to the manifest.
+                    </span>
+                  </div>
+                  <div class="form-group">
+                    <label for="manifest_url" class="col-sm-2 control-label text-left">URL</label>
+                    <div class="col-sm-10">
+                      <input
+                        type="text"
+                        class="form-control"
+                        ng-model="ctrl.formValues.ManifestURL"
+                        id="manifest_url"
+                        placeholder="https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml"
+                        data-cy="k8sAppDeploy-urlFileUrl"
+                      />
+                    </div>
+                  </div>
+                </div>                
+                <!-- !url -->
+
                 <!-- actions -->
                 <div class="col-sm-12 form-section-title">
                   Actions

app/kubernetes/views/deploy/deployController.js

@@ -1,6 +1,7 @@
 import angular from 'angular';
 import _ from 'lodash-es';
 import stripAnsi from 'strip-ansi';
+import PortainerError from 'Portainer/error';
 
 import { KubernetesDeployManifestTypes, KubernetesDeployBuildMethods, KubernetesDeployRequestMethods } from 'Kubernetes/models/deploy';
 import { buildOption } from '@/portainer/components/box-selector';
@@ -24,6 +25,7 @@ class KubernetesDeployController {
     this.methodOptions = [
       buildOption('method_repo', 'fab fa-github', 'Git Repository', 'Use a git repository', KubernetesDeployBuildMethods.GIT),
       buildOption('method_editor', 'fa fa-edit', 'Web editor', 'Use our Web editor', KubernetesDeployBuildMethods.WEB_EDITOR),
+      buildOption('method_url', 'fa fa-globe', 'URL', 'Specify a URL to a file', KubernetesDeployBuildMethods.URL),
     ];
 
     this.state = {
@@ -54,10 +56,11 @@ class KubernetesDeployController {
       this.state.BuildMethod === KubernetesDeployBuildMethods.GIT &&
       (!this.formValues.RepositoryURL ||
         !this.formValues.FilePathInRepository ||
-        (this.formValues.RepositoryAuthentication && (!this.formValues.RepositoryUsername || !this.formValues.RepositoryPassword)));
-    const isWebEditorInvalid = this.state.BuildMethod === KubernetesDeployBuildMethods.WEB_EDITOR && _.isEmpty(this.formValues.EditorContent);
+        (this.formValues.RepositoryAuthentication && (!this.formValues.RepositoryUsername || !this.formValues.RepositoryPassword))) && _.isEmpty(this.formValues.Namespace);
+    const isWebEditorInvalid = this.state.BuildMethod === KubernetesDeployBuildMethods.WEB_EDITOR && _.isEmpty(this.formValues.EditorContent) && _.isEmpty(this.formValues.Namespace);
+    const isURLFormInvalid = this.state.BuildMethod == KubernetesDeployBuildMethods.WEB_EDITOR.URL && _.isEmpty(this.formValues.ManifestURL);
 
-    return isGitFormInvalid || isWebEditorInvalid || _.isEmpty(this.formValues.Namespace) || this.state.actionInProgress;
+    return isGitFormInvalid || isWebEditorInvalid || isURLFormInvalid || this.state.actionInProgress;
   }
 
   onChangeFormValues(values) {
@@ -91,7 +94,20 @@ class KubernetesDeployController {
     this.state.actionInProgress = true;
 
     try {
-      const method = this.state.BuildMethod === this.BuildMethods.GIT ? KubernetesDeployRequestMethods.REPOSITORY : KubernetesDeployRequestMethods.STRING;
+      let method;
+      switch (this.state.BuildMethod) {
+        case this.BuildMethods.GIT:
+          method = KubernetesDeployRequestMethods.REPOSITORY;
+          break;
+        case this.BuildMethods.WEB_EDITOR:
+          method = KubernetesDeployRequestMethods.STRING;
+          break;
+        case this.BuildMethods.URL:
+          method = KubernetesDeployRequestMethods.URL;
+          break;
+        default:
+          throw new PortainerError('Unable to determine build method');
+      }
 
       const payload = {
         ComposeFormat: this.state.DeployType === this.ManifestDeployTypes.COMPOSE,
@@ -107,8 +123,11 @@ class KubernetesDeployController {
           payload.RepositoryPassword = this.formValues.RepositoryPassword;
         }
         payload.FilePathInRepository = this.formValues.FilePathInRepository;
-      } else {
+      } else if (method === KubernetesDeployRequestMethods.WEB_EDITOR) {
         payload.StackFileContent = this.formValues.EditorContent;
+      } else {
+        payload.ManifestURL = this.formValues.ManifestURL;
+        delete payload.Namespace;
       }
 
       await this.StackService.kubernetesDeploy(this.endpointId, method, payload);

build/linux/dev-toolkit/run.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+# Script used to init the Portainer development environment inside the dev-toolkit image
+
+### COLOR OUTPUT ###
+
+ESeq="\x1b["
+RCol="$ESeq"'0m'    # Text Reset
+
+# Regular               Bold                    Underline               High Intensity          BoldHigh Intens         Background              High Intensity Backgrounds
+Bla="$ESeq"'0;30m';     BBla="$ESeq"'1;30m';    UBla="$ESeq"'4;30m';    IBla="$ESeq"'0;90m';    BIBla="$ESeq"'1;90m';   On_Bla="$ESeq"'40m';    On_IBla="$ESeq"'0;100m';
+Red="$ESeq"'0;31m';     BRed="$ESeq"'1;31m';    URed="$ESeq"'4;31m';    IRed="$ESeq"'0;91m';    BIRed="$ESeq"'1;91m';   On_Red="$ESeq"'41m';    On_IRed="$ESeq"'0;101m';
+Gre="$ESeq"'0;32m';     BGre="$ESeq"'1;32m';    UGre="$ESeq"'4;32m';    IGre="$ESeq"'0;92m';    BIGre="$ESeq"'1;92m';   On_Gre="$ESeq"'42m';    On_IGre="$ESeq"'0;102m';
+Yel="$ESeq"'0;33m';     BYel="$ESeq"'1;33m';    UYel="$ESeq"'4;33m';    IYel="$ESeq"'0;93m';    BIYel="$ESeq"'1;93m';   On_Yel="$ESeq"'43m';    On_IYel="$ESeq"'0;103m';
+Blu="$ESeq"'0;34m';     BBlu="$ESeq"'1;34m';    UBlu="$ESeq"'4;34m';    IBlu="$ESeq"'0;94m';    BIBlu="$ESeq"'1;94m';   On_Blu="$ESeq"'44m';    On_IBlu="$ESeq"'0;104m';
+Pur="$ESeq"'0;35m';     BPur="$ESeq"'1;35m';    UPur="$ESeq"'4;35m';    IPur="$ESeq"'0;95m';    BIPur="$ESeq"'1;95m';   On_Pur="$ESeq"'45m';    On_IPur="$ESeq"'0;105m';
+Cya="$ESeq"'0;36m';     BCya="$ESeq"'1;36m';    UCya="$ESeq"'4;36m';    ICya="$ESeq"'0;96m';    BICya="$ESeq"'1;96m';   On_Cya="$ESeq"'46m';    On_ICya="$ESeq"'0;106m';
+Whi="$ESeq"'0;37m';     BWhi="$ESeq"'1;37m';    UWhi="$ESeq"'4;37m';    IWhi="$ESeq"'0;97m';    BIWhi="$ESeq"'1;97m';   On_Whi="$ESeq"'47m';    On_IWhi="$ESeq"'0;107m';
+
+printSection() {
+  echo -e "${BIYel}>>>> ${BIWhi}${1}${RCol}"
+}
+
+info() {
+  echo -e "${BIWhi}${1}${RCol}"
+}
+
+success() {
+  echo -e "${BIGre}${1}${RCol}"
+}
+
+error() {
+  echo -e "${BIRed}${1}${RCol}"
+}
+
+errorAndExit() {
+  echo -e "${BIRed}${1}${RCol}"
+  exit 1
+}
+
+### !COLOR OUTPUT ###
+
+SETUP_FILE=/setup-done
+
+display_configuration() {
+  info "Portainer dev-toolkit container configuration"
+  info "Go version"
+  /usr/local/go/bin/go version
+  info "Node version"
+  node -v
+  info "Yarn version"
+  yarn -v
+  info "Docker version"
+  docker version
+}
+
+main() {
+    [[ -z $PUSER ]] && errorAndExit "Unable to find PUSER environment variable. Please ensure PUSER is set before running this script."
+    [[ -z $PUID ]] && errorAndExit "Unable to find PUID environment variable. Please ensure PUID is set before running this script."
+    [[ -z $PGID ]] && errorAndExit "Unable to find PGID environment variable. Please ensure PGID is set before running this script."
+    [[ -z $DOCKERGID ]] && errorAndExit "Unable to find DOCKERGID environment variable. Please ensure DOCKERGID is set before running this script."
+
+    if [[ -f "${SETUP_FILE}" ]]; then
+        info "Portainer dev-toolkit container already configured."
+        display_configuration
+    else
+        info "Creating user group..."
+        groupadd -g $PGID $PUSER
+
+        info "Creating user..."
+        useradd -l -u $PUID -g $PUSER $PUSER
+
+        info "Setting up home..."
+        install -d -m 0755 -o $PUSER -g $PUSER /home/$PUSER
+
+        info "Configuring Docker..."
+        groupadd -g $DOCKERGID docker
+        usermod -aG docker $PUSER
+
+        info "Configuring Go..."
+        echo "PATH=\"$PATH:/usr/local/go/bin\"" > /etc/environment
+
+        info "Configuring Git..."
+        su $PUSER -c "git config --global url.git@github.com:.insteadOf https://github.com/"
+
+        info "Configuring SSH..."
+        mkdir /home/$PUSER/.ssh
+        cp /host-ssh/* /home/$PUSER/.ssh/
+        chown -R $PUSER:$PUSER /home/$PUSER/.ssh
+
+        touch "${SETUP_FILE}"
+        success "Portainer dev-toolkit container successfully configured."
+
+        display_configuration
+    fi
+}
+
+main
+su $PUSER -s "$@"

build/linux/dev-toolkit/toolkit.Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu
+FROM ubuntu:20.04
 
 # Expose port for the Portainer UI and Edge server
 EXPOSE 9000
@@ -14,13 +14,30 @@ ARG GO_VERSION=go1.16.6.linux-amd64
 
 # Install packages
 RUN apt-get update --fix-missing && apt-get install -qq \
-	dialog \
-	apt-utils \
-	curl \
-	build-essential \
-	nodejs \
-	git \
-	wget
+    dialog \
+    apt-utils \
+    curl \
+    build-essential \
+    git \
+    wget \
+    apt-transport-https \
+    ca-certificates \
+    gnupg-agent \
+    software-properties-common
+
+# Install Docker CLI
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
+    && add-apt-repository \
+   "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+   $(lsb_release -cs) \
+   stable" \
+   && apt-get update \
+   && apt-get install -y docker-ce-cli
+
+
+# Install NodeJS
+RUN curl -fsSL https://deb.nodesource.com/setup_14.x | bash - \
+    && apt-get install -y nodejs
 
 # Install Yarn
 RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
@@ -33,8 +50,8 @@ RUN cd /tmp \
 	&& tar -xf ${GO_VERSION}.tar.gz \
 	&& mv go /usr/local
 
-# Configure Go
-ENV PATH "$PATH:/usr/local/go/bin"
+# Copy run script
+COPY run.sh /
+RUN chmod +x /run.sh
 
-# Confirm installation
-RUN go version && node -v && yarn -v
+ENTRYPOINT ["/run.sh"]

gruntfile.js

@@ -7,6 +7,7 @@ var arch = os.arch();
 if (arch === 'x64') arch = 'amd64';
 
 var portainer_data = '${PORTAINER_DATA:-/tmp/portainer}';
+var portainer_root = process.env.PORTAINER_PROJECT ? process.env.PORTAINER_PROJECT : process.env.PWD;
 
 module.exports = function (grunt) {
   loadGruntTasks(grunt, {
@@ -174,7 +175,7 @@ function shell_build_binary_azuredevops(p, a) {
 function shell_run_container() {
   return [
     'docker rm -f portainer',
-    'docker run -d -p 8000:8000 -p 9000:9000 -v $(pwd)/dist:/app -v ' +
+    'docker run -d --network host -p 8000:8000 -p 9000:9000 -v ' + portainer_root + '/dist:/app -v ' +
       portainer_data +
       ':/data -v /var/run/docker.sock:/var/run/docker.sock:z -v /var/run/docker.sock:/var/run/alternative.sock:z -v /tmp:/tmp --name portainer portainer/base /app/portainer',
   ].join(';');