diff --git a/api/backup/backup.go b/api/backup/backup.go index 3da032c2c..0a62786f5 100644 --- a/api/backup/backup.go +++ b/api/backup/backup.go @@ -10,6 +10,7 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/archive" "github.com/portainer/portainer/api/crypto" + "github.com/portainer/portainer/api/filesystem" "github.com/portainer/portainer/api/http/offlinegate" ) @@ -32,7 +33,7 @@ func CreateBackupArchive(password string, gate *offlinegate.OfflineGate, datasto } for _, filename := range filesToBackup { - err := copyPath(filepath.Join(filestorePath, filename), backupDirPath) + err := filesystem.CopyPath(filepath.Join(filestorePath, filename), backupDirPath) if err != nil { return "", errors.Wrap(err, "Failed to create backup file") } diff --git a/api/backup/copy_test.go b/api/backup/copy_test.go deleted file mode 100644 index b9ceaeaab..000000000 --- a/api/backup/copy_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package backup - -import ( - "io/ioutil" - "os" - "path" - "path/filepath" - "testing" - - "github.com/docker/docker/pkg/ioutils" - "github.com/stretchr/testify/assert" -) - -func listFiles(dir string) []string { - items := make([]string, 0) - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if path == dir { - return nil - } - items = append(items, path) - return nil - }) - - return items -} - -func contains(t *testing.T, list []string, path string) { - assert.Contains(t, list, path) - copyContent, _ := ioutil.ReadFile(path) - assert.Equal(t, "content\n", string(copyContent)) -} - -func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) { - tmpdir, _ := ioutils.TempDir("", "backup") - defer os.RemoveAll(tmpdir) - - err := copyFile("does-not-exist", tmpdir) - assert.NotNil(t, err) -} - -func Test_copyFile_shouldMakeAbackup(t *testing.T) { - tmpdir, _ := ioutils.TempDir("", "backup") - defer os.RemoveAll(tmpdir) - - content := []byte("content") - ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600) - - err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy")) - assert.Nil(t, err) - - copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy")) - assert.Equal(t, content, copyContent) -} - -func Test_copyDir_shouldCopyAllFilesAndDirectories(t *testing.T) { - destination, _ := ioutils.TempDir("", "destination") - defer os.RemoveAll(destination) - err := copyDir("./test_assets/copy_test", destination) - assert.Nil(t, err) - - createdFiles := listFiles(destination) - - contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer")) - contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile")) - contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner")) -} - -func Test_backupPath_shouldSkipWhenNotExist(t *testing.T) { - tmpdir, _ := ioutils.TempDir("", "backup") - defer os.RemoveAll(tmpdir) - - err := copyPath("does-not-exists", tmpdir) - assert.Nil(t, err) - - assert.Empty(t, listFiles(tmpdir)) -} - -func Test_backupPath_shouldCopyFile(t *testing.T) { - tmpdir, _ := ioutils.TempDir("", "backup") - defer os.RemoveAll(tmpdir) - - content := []byte("content") - ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600) - - os.MkdirAll(path.Join(tmpdir, "backup"), 0700) - err := copyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup")) - assert.Nil(t, err) - - copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file")) - assert.Nil(t, err) - assert.Equal(t, content, copyContent) -} - -func Test_backupPath_shouldCopyDir(t *testing.T) { - destination, _ := ioutils.TempDir("", "destination") - defer os.RemoveAll(destination) - err := copyPath("./test_assets/copy_test", destination) - assert.Nil(t, err) - - createdFiles := listFiles(destination) - - contains(t, createdFiles, filepath.Join(destination, "copy_test", "outer")) - contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", ".dotfile")) - contains(t, createdFiles, filepath.Join(destination, "copy_test", "dir", "inner")) -} diff --git a/api/backup/restore.go b/api/backup/restore.go index b0d7acee2..e5329e913 100644 --- a/api/backup/restore.go +++ b/api/backup/restore.go @@ -11,6 +11,7 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/archive" "github.com/portainer/portainer/api/crypto" + "github.com/portainer/portainer/api/filesystem" "github.com/portainer/portainer/api/http/offlinegate" ) @@ -59,7 +60,7 @@ func extractArchive(r io.Reader, destinationDirPath string) error { func restoreFiles(srcDir string, destinationDir string) error { for _, filename := range filesToRestore { - err := copyPath(filepath.Join(srcDir, filename), destinationDir) + err := filesystem.CopyPath(filepath.Join(srcDir, filename), destinationDir) if err != nil { return err } diff --git a/api/bolt/migrator/migrate_dbversion33.go b/api/bolt/migrator/migrate_dbversion33.go new file mode 100644 index 000000000..d7277ada7 --- /dev/null +++ b/api/bolt/migrator/migrate_dbversion33.go @@ -0,0 +1,32 @@ +package migrator + +import ( + portainer "github.com/portainer/portainer/api" +) + +func (m *Migrator) migrateDBVersionTo33() error { + err := migrateStackEntryPoint(m.stackService) + if err != nil { + return err + } + + return nil +} + +func migrateStackEntryPoint(stackService portainer.StackService) error { + stacks, err := stackService.Stacks() + if err != nil { + return err + } + for i := range stacks { + stack := &stacks[i] + if stack.GitConfig == nil { + continue + } + stack.GitConfig.ConfigFilePath = stack.EntryPoint + if err := stackService.UpdateStack(stack.ID, stack); err != nil { + return err + } + } + return nil +} diff --git a/api/bolt/migrator/migrate_dbversion33_test.go b/api/bolt/migrator/migrate_dbversion33_test.go new file mode 100644 index 000000000..256cc121e --- /dev/null +++ b/api/bolt/migrator/migrate_dbversion33_test.go @@ -0,0 +1,51 @@ +package migrator + +import ( + "path" + "testing" + "time" + + "github.com/boltdb/bolt" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/bolt/internal" + "github.com/portainer/portainer/api/bolt/stack" + gittypes "github.com/portainer/portainer/api/git/types" + "github.com/stretchr/testify/assert" +) + +func TestMigrateStackEntryPoint(t *testing.T) { + dbConn, err := bolt.Open(path.Join(t.TempDir(), "portainer-ee-mig-33.db"), 0600, &bolt.Options{Timeout: 1 * time.Second}) + assert.NoError(t, err, "failed to init testing DB connection") + defer dbConn.Close() + + stackService, err := stack.NewService(&internal.DbConnection{DB: dbConn}) + assert.NoError(t, err, "failed to init testing Stack service") + + stacks := []*portainer.Stack{ + { + ID: 1, + EntryPoint: "dir/sub/compose.yml", + }, + { + ID: 2, + EntryPoint: "dir/sub/compose.yml", + GitConfig: &gittypes.RepoConfig{}, + }, + } + + for _, s := range stacks { + err := stackService.CreateStack(s) + assert.NoError(t, err, "failed to create stack") + } + + err = migrateStackEntryPoint(stackService) + assert.NoError(t, err, "failed to migrate entry point to Git ConfigFilePath") + + s, err := stackService.Stack(1) + assert.NoError(t, err) + assert.Nil(t, s.GitConfig, "first stack should not have git config") + + s, err = stackService.Stack(2) + assert.NoError(t, err) + assert.Equal(t, "dir/sub/compose.yml", s.GitConfig.ConfigFilePath, "second stack should have config file path migrated") +} diff --git a/api/bolt/migrator/migrator.go b/api/bolt/migrator/migrator.go index fc00578f0..df3ad0436 100644 --- a/api/bolt/migrator/migrator.go +++ b/api/bolt/migrator/migrator.go @@ -381,5 +381,11 @@ func (m *Migrator) Migrate() error { } } + if m.currentDBVersion < 33 { + if err := m.migrateDBVersionTo33(); err != nil { + return err + } + } + return m.versionService.StoreDBVersion(portainer.DBVersion) } diff --git a/api/bolt/stack/stack.go b/api/bolt/stack/stack.go index f9cfafad7..094a52b2e 100644 --- a/api/bolt/stack/stack.go +++ b/api/bolt/stack/stack.go @@ -1,11 +1,14 @@ package stack import ( + "strings" + portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/bolt/internal" "github.com/boltdb/bolt" + pkgerrors "github.com/pkg/errors" ) const ( @@ -133,3 +136,76 @@ func (service *Service) DeleteStack(ID portainer.StackID) error { identifier := internal.Itob(int(ID)) return internal.DeleteObject(service.connection, BucketName, identifier) } + +// StackByWebhookID returns a pointer to a stack object by webhook ID. +// It returns nil, errors.ErrObjectNotFound if there's no stack associated with the webhook ID. +func (service *Service) StackByWebhookID(id string) (*portainer.Stack, error) { + if id == "" { + return nil, pkgerrors.New("webhook ID can't be empty string") + } + var stack portainer.Stack + found := false + + err := service.connection.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(BucketName)) + cursor := bucket.Cursor() + + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + var t struct { + AutoUpdate *struct { + WebhookID string `json:"Webhook"` + } `json:"AutoUpdate"` + } + + err := internal.UnmarshalObject(v, &t) + if err != nil { + return err + } + + if t.AutoUpdate != nil && strings.EqualFold(t.AutoUpdate.WebhookID, id) { + found = true + err := internal.UnmarshalObject(v, &stack) + if err != nil { + return err + } + break + } + } + + return nil + }) + + if err != nil { + return nil, err + } + if !found { + return nil, errors.ErrObjectNotFound + } + + return &stack, nil +} + +// RefreshableStacks returns stacks that are configured for a periodic update +func (service *Service) RefreshableStacks() ([]portainer.Stack, error) { + stacks := make([]portainer.Stack, 0) + err := service.connection.View(func(tx *bolt.Tx) error { + bucket := tx.Bucket([]byte(BucketName)) + cursor := bucket.Cursor() + + var stack portainer.Stack + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + err := internal.UnmarshalObject(v, &stack) + if err != nil { + return err + } + + if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" { + stacks = append(stacks, stack) + } + } + + return nil + }) + + return stacks, err +} diff --git a/api/bolt/stack/tests/stack_test.go b/api/bolt/stack/tests/stack_test.go new file mode 100644 index 000000000..d0c66dadf --- /dev/null +++ b/api/bolt/stack/tests/stack_test.go @@ -0,0 +1,111 @@ +package tests + +import ( + "testing" + "time" + + "github.com/portainer/portainer/api/bolt" + + bolterrors "github.com/portainer/portainer/api/bolt/errors" + + "github.com/portainer/portainer/api/bolt/bolttest" + + "github.com/gofrs/uuid" + + "github.com/stretchr/testify/assert" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/filesystem" +) + +func newGuidString(t *testing.T) string { + uuid, err := uuid.NewV4() + assert.NoError(t, err) + + return uuid.String() +} + +type stackBuilder struct { + t *testing.T + count int + store *bolt.Store +} + +func TestService_StackByWebhookID(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode. Normally takes ~1s to run.") + } + store, teardown := bolttest.MustNewTestStore(true) + defer teardown() + + b := stackBuilder{t: t, store: store} + b.createNewStack(newGuidString(t)) + for i := 0; i < 10; i++ { + b.createNewStack("") + } + webhookID := newGuidString(t) + stack := b.createNewStack(webhookID) + + // can find a stack by webhook ID + got, err := store.StackService.StackByWebhookID(webhookID) + assert.NoError(t, err) + assert.Equal(t, stack, *got) + + // returns nil and object not found error if there's no stack associated with the webhook + got, err = store.StackService.StackByWebhookID(newGuidString(t)) + assert.Nil(t, got) + assert.ErrorIs(t, err, bolterrors.ErrObjectNotFound) +} + +func (b *stackBuilder) createNewStack(webhookID string) portainer.Stack { + b.count++ + stack := portainer.Stack{ + ID: portainer.StackID(b.count), + Name: "Name", + Type: portainer.DockerComposeStack, + EndpointID: 2, + EntryPoint: filesystem.ComposeFileDefaultName, + Env: []portainer.Pair{{"Name1", "Value1"}}, + Status: portainer.StackStatusActive, + CreationDate: time.Now().Unix(), + ProjectPath: "/tmp/project", + CreatedBy: "test", + } + + if webhookID == "" { + if b.count%2 == 0 { + stack.AutoUpdate = &portainer.StackAutoUpdate{ + Interval: "", + Webhook: "", + } + } // else keep AutoUpdate nil + } else { + stack.AutoUpdate = &portainer.StackAutoUpdate{Webhook: webhookID} + } + + err := b.store.StackService.CreateStack(&stack) + assert.NoError(b.t, err) + + return stack +} + +func Test_RefreshableStacks(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode. Normally takes ~1s to run.") + } + store, teardown := bolttest.MustNewTestStore(true) + defer teardown() + + staticStack := portainer.Stack{ID: 1} + stackWithWebhook := portainer.Stack{ID: 2, AutoUpdate: &portainer.StackAutoUpdate{Webhook: "webhook"}} + refreshableStack := portainer.Stack{ID: 3, AutoUpdate: &portainer.StackAutoUpdate{Interval: "1m"}} + + for _, stack := range []*portainer.Stack{&staticStack, &stackWithWebhook, &refreshableStack} { + err := store.Stack().CreateStack(stack) + assert.NoError(t, err) + } + + stacks, err := store.Stack().RefreshableStacks() + assert.NoError(t, err) + assert.ElementsMatch(t, []portainer.Stack{refreshableStack}, stacks) +} diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index e22ef13c2..06a004453 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -6,7 +6,6 @@ import ( "os" "strings" - wrapper "github.com/portainer/docker-compose-wrapper" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/bolt" "github.com/portainer/portainer/api/chisel" @@ -30,6 +29,8 @@ import ( "github.com/portainer/portainer/api/ldap" "github.com/portainer/portainer/api/libcompose" "github.com/portainer/portainer/api/oauth" + "github.com/portainer/portainer/api/scheduler" + "github.com/portainer/portainer/api/stacks" ) func initCLI() *portainer.CLIFlags { @@ -80,12 +81,8 @@ func initDataStore(dataStorePath string, fileService portainer.FileService) port func initComposeStackManager(assetsPath string, dataStorePath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager { composeWrapper, err := exec.NewComposeStackManager(assetsPath, dataStorePath, proxyManager) if err != nil { - if err == wrapper.ErrBinaryNotFound { - log.Printf("[INFO] [message: docker-compose binary not found, falling back to libcompose]") - return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService) - } - - log.Fatalf("failed initalizing compose stack manager; err=%s", err) + log.Printf("[INFO] [main,compose] [message: falling-back to libcompose] [error: %s]", err) + return libcompose.NewComposeStackManager(dataStorePath, reverseTunnelService) } return composeWrapper @@ -470,6 +467,10 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { log.Fatalf("failed starting license service: %s", err) } + scheduler := scheduler.NewScheduler(shutdownCtx) + stackDeployer := stacks.NewStackDeployer(swarmStackManager, composeStackManager) + stacks.StartStackSchedules(scheduler, stackDeployer, dataStore, gitService) + return &http.Server{ AuthorizationService: authorizationService, ReverseTunnelService: reverseTunnelService, @@ -495,8 +496,10 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { SSLKey: *flags.SSLKey, DockerClientFactory: dockerClientFactory, KubernetesClientFactory: kubernetesClientFactory, + Scheduler: scheduler, ShutdownCtx: shutdownCtx, ShutdownTrigger: shutdownTrigger, + StackDeployer: stackDeployer, } } diff --git a/api/exec/compose_stack.go b/api/exec/compose_stack.go index 1b527478f..1f4bcc301 100644 --- a/api/exec/compose_stack.go +++ b/api/exec/compose_stack.go @@ -11,17 +11,18 @@ import ( portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/http/proxy" "github.com/portainer/portainer/api/http/proxy/factory" + "github.com/portainer/portainer/api/internal/stackutils" ) // ComposeStackManager is a wrapper for docker-compose binary type ComposeStackManager struct { wrapper *wrapper.ComposeWrapper - configPath string + dataPath string proxyManager *proxy.Manager } // NewComposeStackManager returns a docker-compose wrapper if corresponding binary present, otherwise nil -func NewComposeStackManager(binaryPath string, configPath string, proxyManager *proxy.Manager) (*ComposeStackManager, error) { +func NewComposeStackManager(binaryPath string, dataPath string, proxyManager *proxy.Manager) (*ComposeStackManager, error) { wrap, err := wrapper.NewComposeWrapper(binaryPath) if err != nil { return nil, err @@ -30,15 +31,10 @@ func NewComposeStackManager(binaryPath string, configPath string, proxyManager * return &ComposeStackManager{ wrapper: wrap, proxyManager: proxyManager, - configPath: configPath, + dataPath: dataPath, }, nil } -// NormalizeStackName returns a new stack name with unsupported characters replaced -func (w *ComposeStackManager) NormalizeStackName(name string) string { - return name -} - // ComposeSyntaxMaxVersion returns the maximum supported version of the docker compose syntax func (w *ComposeStackManager) ComposeSyntaxMaxVersion() string { return portainer.ComposeSyntaxMaxVersion @@ -60,9 +56,8 @@ func (w *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portainer.End return err } - filePath := stackFilePath(stack) - - _, err = w.wrapper.Up([]string{filePath}, url, stack.Name, envFilePath, w.configPath) + filePaths := stackutils.GetStackFilePaths(stack) + _, err = w.wrapper.Up(filePaths, url, stack.Name, envFilePath, w.dataPath) return err } @@ -76,14 +71,15 @@ func (w *ComposeStackManager) Down(stack *portainer.Stack, endpoint *portainer.E defer proxy.Close() } - filePath := stackFilePath(stack) + filePaths := stackutils.GetStackFilePaths(stack) - _, err = w.wrapper.Down([]string{filePath}, url, stack.Name) + _, err = w.wrapper.Down(filePaths, url, stack.Name) return err } -func stackFilePath(stack *portainer.Stack) string { - return path.Join(stack.ProjectPath, stack.EntryPoint) +// NormalizeStackName returns the passed stack name, for interface implementation only +func (w *ComposeStackManager) NormalizeStackName(name string) string { + return name } func (w *ComposeStackManager) fetchEndpointProxy(endpoint *portainer.Endpoint) (string, *factory.ProxyServer, error) { diff --git a/api/exec/compose_stack_integration_test.go b/api/exec/compose_stack_integration_test.go index 8214d3461..f0b279659 100644 --- a/api/exec/compose_stack_integration_test.go +++ b/api/exec/compose_stack_integration_test.go @@ -1,5 +1,3 @@ -// +build integration - package exec import ( diff --git a/api/exec/compose_stack_test.go b/api/exec/compose_stack_test.go index 80e2818df..caa4728a6 100644 --- a/api/exec/compose_stack_test.go +++ b/api/exec/compose_stack_test.go @@ -10,47 +10,6 @@ import ( "github.com/stretchr/testify/assert" ) -func Test_stackFilePath(t *testing.T) { - tests := []struct { - name string - stack *portainer.Stack - expected string - }{ - // { - // name: "should return empty result if stack is missing", - // stack: nil, - // expected: "", - // }, - // { - // name: "should return empty result if stack don't have entrypoint", - // stack: &portainer.Stack{}, - // expected: "", - // }, - { - name: "should allow file name and dir", - stack: &portainer.Stack{ - ProjectPath: "dir", - EntryPoint: "file", - }, - expected: path.Join("dir", "file"), - }, - { - name: "should allow file name only", - stack: &portainer.Stack{ - EntryPoint: "file", - }, - expected: "file", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := stackFilePath(tt.stack) - assert.Equal(t, tt.expected, result) - }) - } -} - func Test_createEnvFile(t *testing.T) { dir := t.TempDir() @@ -60,11 +19,6 @@ func Test_createEnvFile(t *testing.T) { expected string expectedFile bool }{ - // { - // name: "should not add env file option if stack is missing", - // stack: nil, - // expected: "", - // }, { name: "should not add env file option if stack doesn't have env variables", stack: &portainer.Stack{ diff --git a/api/exec/swarm_stack.go b/api/exec/swarm_stack.go index faf6cf723..60da40bb1 100644 --- a/api/exec/swarm_stack.go +++ b/api/exec/swarm_stack.go @@ -11,6 +11,7 @@ import ( "runtime" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/stackutils" ) // SwarmStackManager represents a service for managing stacks. @@ -61,22 +62,23 @@ func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error { // Deploy executes the docker stack deploy command. func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, endpoint *portainer.Endpoint) error { - stackFilePath := path.Join(stack.ProjectPath, stack.EntryPoint) + filePaths := stackutils.GetStackFilePaths(stack) command, args := manager.prepareDockerCommandAndArgs(manager.binaryPath, manager.dataPath, endpoint) if prune { - args = append(args, "stack", "deploy", "--prune", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name) + args = append(args, "stack", "deploy", "--prune", "--with-registry-auth") } else { - args = append(args, "stack", "deploy", "--with-registry-auth", "--compose-file", stackFilePath, stack.Name) + args = append(args, "stack", "deploy", "--with-registry-auth") } + args = configureFilePaths(args, filePaths) + args = append(args, stack.Name) + env := make([]string, 0) for _, envvar := range stack.Env { env = append(env, envvar.Name+"="+envvar.Value) } - - stackFolder := path.Dir(stackFilePath) - return runCommandAndCaptureStdErr(command, args, env, stackFolder) + return runCommandAndCaptureStdErr(command, args, env, stack.ProjectPath) } // Remove executes the docker stack rm command. @@ -184,3 +186,10 @@ func (manager *SwarmStackManager) retrieveConfigurationFromDisk(path string) (ma return config, nil } + +func configureFilePaths(args []string, filePaths []string) []string { + for _, path := range filePaths { + args = append(args, "--compose-file", path) + } + return args +} diff --git a/api/exec/swarm_stack_test.go b/api/exec/swarm_stack_test.go new file mode 100644 index 000000000..47d28ce2c --- /dev/null +++ b/api/exec/swarm_stack_test.go @@ -0,0 +1,15 @@ +package exec + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfigFilePaths(t *testing.T) { + args := []string{"stack", "deploy", "--with-registry-auth"} + filePaths := []string{"dir/file", "dir/file-two", "dir/file-three"} + expected := []string{"stack", "deploy", "--with-registry-auth", "--compose-file", "dir/file", "--compose-file", "dir/file-two", "--compose-file", "dir/file-three"} + output := configureFilePaths(args, filePaths) + assert.ElementsMatch(t, expected, output, "wrong output file paths") +} diff --git a/api/backup/copy.go b/api/filesystem/copy.go similarity index 63% rename from api/backup/copy.go rename to api/filesystem/copy.go index 6aaefd54c..abf4d33aa 100644 --- a/api/backup/copy.go +++ b/api/filesystem/copy.go @@ -1,4 +1,4 @@ -package backup +package filesystem import ( "errors" @@ -8,7 +8,8 @@ import ( "strings" ) -func copyPath(path string, toDir string) error { +// CopyPath copies file or directory defined by the path to the toDir path +func CopyPath(path string, toDir string) error { info, err := os.Stat(path) if err != nil && errors.Is(err, os.ErrNotExist) { // skip copy if file does not exist @@ -20,17 +21,30 @@ func copyPath(path string, toDir string) error { return copyFile(path, destination) } - return copyDir(path, toDir) + return CopyDir(path, toDir, true) } -func copyDir(fromDir, toDir string) error { +// CopyDir copies contents of fromDir to toDir. +// When keepParent is true, contents will be copied with their immediate parent dir, +// i.e. given /from/dirA and /to/dirB with keepParent == true, result will be /to/dirB/dirA/ +func CopyDir(fromDir, toDir string, keepParent bool) error { cleanedSourcePath := filepath.Clean(fromDir) parentDirectory := filepath.Dir(cleanedSourcePath) err := filepath.Walk(cleanedSourcePath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - destination := filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory)) + var destination string + if keepParent { + destination = filepath.Join(toDir, strings.TrimPrefix(path, parentDirectory)) + } else { + destination = filepath.Join(toDir, strings.TrimPrefix(path, cleanedSourcePath)) + } + + if destination == "" { + return nil + } + if info.IsDir() { return nil // skip directory creations } diff --git a/api/filesystem/copy_test.go b/api/filesystem/copy_test.go new file mode 100644 index 000000000..2fcef9e6b --- /dev/null +++ b/api/filesystem/copy_test.go @@ -0,0 +1,92 @@ +package filesystem + +import ( + "io/ioutil" + "os" + "path" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_copyFile_returnsError_whenSourceDoesNotExist(t *testing.T) { + tmpdir, _ := ioutil.TempDir("", "backup") + defer os.RemoveAll(tmpdir) + + err := copyFile("does-not-exist", tmpdir) + assert.Error(t, err) +} + +func Test_copyFile_shouldMakeAbackup(t *testing.T) { + tmpdir, _ := ioutil.TempDir("", "backup") + defer os.RemoveAll(tmpdir) + + content := []byte("content") + ioutil.WriteFile(path.Join(tmpdir, "origin"), content, 0600) + + err := copyFile(path.Join(tmpdir, "origin"), path.Join(tmpdir, "copy")) + assert.NoError(t, err) + + copyContent, _ := ioutil.ReadFile(path.Join(tmpdir, "copy")) + assert.Equal(t, content, copyContent) +} + +func Test_CopyDir_shouldCopyAllFilesAndDirectories(t *testing.T) { + destination, _ := ioutil.TempDir("", "destination") + defer os.RemoveAll(destination) + err := CopyDir("./testdata/copy_test", destination, true) + assert.NoError(t, err) + + assert.FileExists(t, filepath.Join(destination, "copy_test", "outer")) + assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile")) + assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner")) +} + +func Test_CopyDir_shouldCopyOnlyDirContents(t *testing.T) { + destination, _ := ioutil.TempDir("", "destination") + defer os.RemoveAll(destination) + err := CopyDir("./testdata/copy_test", destination, false) + assert.NoError(t, err) + + assert.FileExists(t, filepath.Join(destination, "outer")) + assert.FileExists(t, filepath.Join(destination, "dir", ".dotfile")) + assert.FileExists(t, filepath.Join(destination, "dir", "inner")) +} + +func Test_CopyPath_shouldSkipWhenNotExist(t *testing.T) { + tmpdir, _ := ioutil.TempDir("", "backup") + defer os.RemoveAll(tmpdir) + + err := CopyPath("does-not-exists", tmpdir) + assert.NoError(t, err) + + assert.NoFileExists(t, tmpdir) +} + +func Test_CopyPath_shouldCopyFile(t *testing.T) { + tmpdir, _ := ioutil.TempDir("", "backup") + defer os.RemoveAll(tmpdir) + + content := []byte("content") + ioutil.WriteFile(path.Join(tmpdir, "file"), content, 0600) + + os.MkdirAll(path.Join(tmpdir, "backup"), 0700) + err := CopyPath(path.Join(tmpdir, "file"), path.Join(tmpdir, "backup")) + assert.NoError(t, err) + + copyContent, err := ioutil.ReadFile(path.Join(tmpdir, "backup", "file")) + assert.NoError(t, err) + assert.Equal(t, content, copyContent) +} + +func Test_CopyPath_shouldCopyDir(t *testing.T) { + destination, _ := ioutil.TempDir("", "destination") + defer os.RemoveAll(destination) + err := CopyPath("./testdata/copy_test", destination) + assert.NoError(t, err) + + assert.FileExists(t, filepath.Join(destination, "copy_test", "outer")) + assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", ".dotfile")) + assert.FileExists(t, filepath.Join(destination, "copy_test", "dir", "inner")) +} diff --git a/api/backup/test_assets/copy_test/dir/.dotfile b/api/filesystem/testdata/copy_test/dir/.dotfile similarity index 100% rename from api/backup/test_assets/copy_test/dir/.dotfile rename to api/filesystem/testdata/copy_test/dir/.dotfile diff --git a/api/backup/test_assets/copy_test/dir/inner b/api/filesystem/testdata/copy_test/dir/inner similarity index 100% rename from api/backup/test_assets/copy_test/dir/inner rename to api/filesystem/testdata/copy_test/dir/inner diff --git a/api/backup/test_assets/copy_test/outer b/api/filesystem/testdata/copy_test/outer similarity index 100% rename from api/backup/test_assets/copy_test/outer rename to api/filesystem/testdata/copy_test/outer diff --git a/api/git/azure.go b/api/git/azure.go index 78f10e52d..417d5db4b 100644 --- a/api/git/azure.go +++ b/api/git/azure.go @@ -2,15 +2,17 @@ package git import ( "context" + "encoding/json" "fmt" - "github.com/pkg/errors" - "github.com/portainer/portainer/api/archive" "io" "io/ioutil" "net/http" "net/url" "os" "strings" + + "github.com/pkg/errors" + "github.com/portainer/portainer/api/archive" ) const ( @@ -37,7 +39,7 @@ type azureDownloader struct { func NewAzureDownloader(client *http.Client) *azureDownloader { return &azureDownloader{ - client: client, + client: client, baseUrl: "https://dev.azure.com", } } @@ -100,6 +102,57 @@ func (a *azureDownloader) downloadZipFromAzureDevOps(ctx context.Context, option return zipFile.Name(), nil } +func (a *azureDownloader) latestCommitID(ctx context.Context, options fetchOptions) (string, error) { + config, err := parseUrl(options.repositoryUrl) + if err != nil { + return "", errors.WithMessage(err, "failed to parse url") + } + + refsUrl, err := a.buildRefsUrl(config, options.referenceName) + if err != nil { + return "", errors.WithMessage(err, "failed to build azure refs url") + } + + req, err := http.NewRequestWithContext(ctx, "GET", refsUrl, nil) + if options.username != "" || options.password != "" { + req.SetBasicAuth(options.username, options.password) + } else if config.username != "" || config.password != "" { + req.SetBasicAuth(config.username, config.password) + } + + if err != nil { + return "", errors.WithMessage(err, "failed to create a new HTTP request") + } + + resp, err := a.client.Do(req) + if err != nil { + return "", errors.WithMessage(err, "failed to make an HTTP request") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to get repository refs with a status \"%v\"", resp.Status) + } + + var refs struct { + Value []struct { + Name string `json:"name"` + ObjectId string `json:"objectId"` + } + } + if err := json.NewDecoder(resp.Body).Decode(&refs); err != nil { + return "", errors.Wrap(err, "could not parse Azure Refs response") + } + + for _, ref := range refs.Value { + if strings.EqualFold(ref.Name, options.referenceName) { + return ref.ObjectId, nil + } + } + + return "", errors.Errorf("could not find ref %q in the repository", options.referenceName) +} + func parseUrl(rawUrl string) (*azureOptions, error) { if strings.HasPrefix(rawUrl, "https://") || strings.HasPrefix(rawUrl, "http://") { return parseHttpUrl(rawUrl) @@ -193,6 +246,27 @@ func (a *azureDownloader) buildDownloadUrl(config *azureOptions, referenceName s return u.String(), nil } +func (a *azureDownloader) buildRefsUrl(config *azureOptions, referenceName string) (string, error) { + rawUrl := fmt.Sprintf("%s/%s/%s/_apis/git/repositories/%s/refs", + a.baseUrl, + url.PathEscape(config.organisation), + url.PathEscape(config.project), + url.PathEscape(config.repository)) + u, err := url.Parse(rawUrl) + + if err != nil { + return "", errors.Wrapf(err, "failed to parse refs url path %s", rawUrl) + } + + // filterContains=main&api-version=6.0 + q := u.Query() + q.Set("filterContains", formatReferenceName(referenceName)) + q.Set("api-version", "6.0") + u.RawQuery = q.Encode() + + return u.String(), nil +} + const ( branchPrefix = "refs/heads/" tagPrefix = "refs/tags/" diff --git a/api/git/azure_integration_test.go b/api/git/azure_integration_test.go index 6d684d877..200747daa 100644 --- a/api/git/azure_integration_test.go +++ b/api/git/azure_integration_test.go @@ -78,6 +78,18 @@ func TestService_ClonePrivateRepository_Azure(t *testing.T) { assert.FileExists(t, filepath.Join(dst, "README.md")) } +func TestService_LatestCommitID_Azure(t *testing.T) { + ensureIntegrationTest(t) + + pat := getRequiredValue(t, "AZURE_DEVOPS_PAT") + service := NewService() + + repositoryUrl := "https://portainer.visualstudio.com/Playground/_git/dev_integration" + id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", "", pat) + assert.NoError(t, err) + assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty") +} + func getRequiredValue(t *testing.T, name string) string { value, ok := os.LookupEnv(name) if !ok { diff --git a/api/git/azure_test.go b/api/git/azure_test.go index 18417e9e6..b95dc981e 100644 --- a/api/git/azure_test.go +++ b/api/git/azure_test.go @@ -2,11 +2,12 @@ package git import ( "context" - "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" "net/url" "testing" + + "github.com/stretchr/testify/assert" ) func Test_buildDownloadUrl(t *testing.T) { @@ -27,6 +28,23 @@ func Test_buildDownloadUrl(t *testing.T) { } } +func Test_buildRefsUrl(t *testing.T) { + a := NewAzureDownloader(nil) + u, err := a.buildRefsUrl(&azureOptions{ + organisation: "organisation", + project: "project", + repository: "repository", + }, "refs/heads/main") + + expectedUrl, _ := url.Parse("https://dev.azure.com/organisation/project/_apis/git/repositories/repository/refs?filterContains=main&api-version=6.0") + actualUrl, _ := url.Parse(u) + assert.NoError(t, err) + assert.Equal(t, expectedUrl.Host, actualUrl.Host) + assert.Equal(t, expectedUrl.Scheme, actualUrl.Scheme) + assert.Equal(t, expectedUrl.Path, actualUrl.Path) + assert.Equal(t, expectedUrl.Query(), actualUrl.Query()) +} + func Test_parseAzureUrl(t *testing.T) { type args struct { url string @@ -248,3 +266,110 @@ func Test_azureDownloader_downloadZipFromAzureDevOps(t *testing.T) { }) } } + +func Test_azureDownloader_latestCommitID(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + response := `{ + "value": [ + { + "name": "refs/heads/feature/calcApp", + "objectId": "ffe9cba521f00d7f60e322845072238635edb451", + "creator": { + "displayName": "Normal Paulk", + "url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "_links": { + "avatar": { + "href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + } + }, + "id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "uniqueName": "dev@mailserver.com", + "imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + }, + "url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2FcalcApp" + }, + { + "name": "refs/heads/feature/replacer", + "objectId": "917131a709996c5cfe188c3b57e9a6ad90e8b85c", + "creator": { + "displayName": "Normal Paulk", + "url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "_links": { + "avatar": { + "href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + } + }, + "id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "uniqueName": "dev@mailserver.com", + "imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + }, + "url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Ffeature%2Freplacer" + }, + { + "name": "refs/heads/master", + "objectId": "ffe9cba521f00d7f60e322845072238635edb451", + "creator": { + "displayName": "Normal Paulk", + "url": "https://vssps.dev.azure.com/fabrikam/_apis/Identities/ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "_links": { + "avatar": { + "href": "https://dev.azure.com/fabrikam/_apis/GraphProfile/MemberAvatars/aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + } + }, + "id": "ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "uniqueName": "dev@mailserver.com", + "imageUrl": "https://dev.azure.com/fabrikam/_api/_common/identityImage?id=ac5aaba6-a66a-4e1d-b508-b060ec624fa9", + "descriptor": "aad.YmFjMGYyZDctNDA3ZC03OGRhLTlhMjUtNmJhZjUwMWFjY2U5" + }, + "url": "https://dev.azure.com/fabrikam/7484f783-66a3-4f27-b7cd-6b08b0b077ed/_apis/git/repositories/d3d1760b-311c-4175-a726-20dfc6a7f885/refs?filter=heads%2Fmaster" + } + ], + "count": 3 + }` + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(response)) + })) + defer server.Close() + + a := &azureDownloader{ + client: server.Client(), + baseUrl: server.URL, + } + + tests := []struct { + name string + args fetchOptions + want string + wantErr bool + }{ + { + name: "should be able to parse response", + args: fetchOptions{ + referenceName: "refs/heads/master", + repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"}, + want: "ffe9cba521f00d7f60e322845072238635edb451", + wantErr: false, + }, + { + name: "should be able to parse response", + args: fetchOptions{ + referenceName: "refs/heads/unknown", + repositoryUrl: "https://dev.azure.com/Organisation/Project/_git/Repository"}, + want: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + id, err := a.latestCommitID(context.Background(), tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("azureDownloader.latestCommitID() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.Equal(t, tt.want, id) + }) + } +} diff --git a/api/git/git.go b/api/git/git.go index 7887f7d95..b8bce4dc9 100644 --- a/api/git/git.go +++ b/api/git/git.go @@ -6,16 +6,26 @@ import ( "net/http" "os" "path/filepath" + "strings" "time" "github.com/pkg/errors" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/transport/client" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/storage/memory" ) +type fetchOptions struct { + repositoryUrl string + username string + password string + referenceName string +} + type cloneOptions struct { repositoryUrl string username string @@ -26,6 +36,7 @@ type cloneOptions struct { type downloader interface { download(ctx context.Context, dst string, opt cloneOptions) error + latestCommitID(ctx context.Context, opt fetchOptions) (string, error) } type gitClient struct { @@ -62,6 +73,34 @@ func (c gitClient) download(ctx context.Context, dst string, opt cloneOptions) e return nil } +func (c gitClient) latestCommitID(ctx context.Context, opt fetchOptions) (string, error) { + remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ + Name: "origin", + URLs: []string{opt.repositoryUrl}, + }) + + listOptions := &git.ListOptions{} + if opt.password != "" || opt.username != "" { + listOptions.Auth = &githttp.BasicAuth{ + Username: opt.username, + Password: opt.password, + } + } + + refs, err := remote.List(listOptions) + if err != nil { + return "", errors.Wrap(err, "failed to list repository refs") + } + + for _, ref := range refs { + if strings.EqualFold(ref.Name().String(), opt.referenceName) { + return ref.Hash().String(), nil + } + } + + return "", errors.Errorf("could not find ref %q in the repository", opt.referenceName) +} + // Service represents a service for managing Git. type Service struct { httpsCli *http.Client @@ -108,3 +147,19 @@ func (service *Service) cloneRepository(destination string, options cloneOptions return service.git.download(context.TODO(), destination, options) } + +// LatestCommitID returns SHA1 of the latest commit of the specified reference +func (service *Service) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { + options := fetchOptions{ + repositoryUrl: repositoryURL, + username: username, + password: password, + referenceName: referenceName, + } + + if isAzureUrl(options.repositoryUrl) { + return service.azure.latestCommitID(context.TODO(), options) + } + + return service.git.latestCommitID(context.TODO(), options) +} diff --git a/api/git/git_integration_test.go b/api/git/git_integration_test.go index d35ba8d52..6f123c130 100644 --- a/api/git/git_integration_test.go +++ b/api/git/git_integration_test.go @@ -12,7 +12,7 @@ import ( func TestService_ClonePrivateRepository_GitHub(t *testing.T) { ensureIntegrationTest(t) - pat := getRequiredValue(t, "GITHUB_PAT") + accessToken := getRequiredValue(t, "GITHUB_PAT") username := getRequiredValue(t, "GITHUB_USERNAME") service := NewService() @@ -21,7 +21,20 @@ func TestService_ClonePrivateRepository_GitHub(t *testing.T) { defer os.RemoveAll(dst) repositoryUrl := "https://github.com/portainer/private-test-repository.git" - err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, pat) + err = service.CloneRepository(dst, repositoryUrl, "refs/heads/main", username, accessToken) assert.NoError(t, err) assert.FileExists(t, filepath.Join(dst, "README.md")) } + +func TestService_LatestCommitID_GitHub(t *testing.T) { + ensureIntegrationTest(t) + + accessToken := getRequiredValue(t, "GITHUB_PAT") + username := getRequiredValue(t, "GITHUB_USERNAME") + service := NewService() + + repositoryUrl := "https://github.com/portainer/private-test-repository.git" + id, err := service.LatestCommitID(repositoryUrl, "refs/heads/main", username, accessToken) + assert.NoError(t, err) + assert.NotEmpty(t, id, "cannot guarantee commit id, but it should be not empty") +} diff --git a/api/git/git_test.go b/api/git/git_test.go index 14878b304..11ff5e5c8 100644 --- a/api/git/git_test.go +++ b/api/git/git_test.go @@ -105,7 +105,19 @@ func Test_cloneRepository(t *testing.T) { }) assert.NoError(t, err) - assert.Equal(t, 3, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth") + assert.Equal(t, 4, getCommitHistoryLength(t, err, dir), "cloned repo has incorrect depth") +} + +func Test_latestCommitID(t *testing.T) { + service := Service{git: gitClient{preserveGitDirectory: true}} // no need for http client since the test access the repo via file system. + + repositoryURL := bareRepoDir + referenceName := "refs/heads/main" + + id, err := service.LatestCommitID(repositoryURL, referenceName, "", "") + + assert.NoError(t, err) + assert.Equal(t, "68dcaa7bd452494043c64252ab90db0f98ecf8d2", id) } func getCommitHistoryLength(t *testing.T, err error, dir string) int { @@ -137,6 +149,10 @@ func (t *testDownloader) download(_ context.Context, _ string, _ cloneOptions) e return nil } +func (t *testDownloader) latestCommitID(_ context.Context, _ fetchOptions) (string, error) { + return "", nil +} + func Test_cloneRepository_azure(t *testing.T) { tests := []struct { name string diff --git a/api/git/testdata/azure-repo copy.zip b/api/git/testdata/azure-repo copy.zip new file mode 100644 index 000000000..d4b53d0b8 Binary files /dev/null and b/api/git/testdata/azure-repo copy.zip differ diff --git a/api/git/testdata/test-clone-git-repo.tar.gz b/api/git/testdata/test-clone-git-repo.tar.gz index ca63d337c..cba76a0d8 100644 Binary files a/api/git/testdata/test-clone-git-repo.tar.gz and b/api/git/testdata/test-clone-git-repo.tar.gz differ diff --git a/api/git/types/types.go b/api/git/types/types.go index 2a91f61b6..df534c69e 100644 --- a/api/git/types/types.go +++ b/api/git/types/types.go @@ -1,7 +1,19 @@ package gittypes +// RepoConfig represents a configuration for a repo type RepoConfig struct { - URL string - ReferenceName string - ConfigFilePath string + // The repo url + URL string `example:"https://github.com/portainer/portainer.git"` + // The reference name + ReferenceName string `example:"refs/heads/branch_name"` + // Path to where the config file is in this url/refName + ConfigFilePath string `example:"docker-compose.yml"` + // Git credentials + Authentication *GitAuthentication + ConfigHash string +} + +type GitAuthentication struct { + Username string + Password string } diff --git a/api/go.mod b/api/go.mod index 71d304240..1edf1b571 100644 --- a/api/go.mod +++ b/api/go.mod @@ -31,6 +31,7 @@ require ( github.com/portainer/libcompose v0.5.3 github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2 github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33 + github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.7.0 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 diff --git a/api/go.sum b/api/go.sum index 4511f7c23..eb9eb8d6b 100644 --- a/api/go.sum +++ b/api/go.sum @@ -264,6 +264,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= diff --git a/api/http/handler/stacks/create_compose_stack.go b/api/http/handler/stacks/create_compose_stack.go index f66c2572d..721fde2eb 100644 --- a/api/http/handler/stacks/create_compose_stack.go +++ b/api/http/handler/stacks/create_compose_stack.go @@ -1,19 +1,22 @@ package stacks import ( - "errors" "fmt" + "log" "net/http" "path" "strconv" "time" "github.com/asaskevich/govalidator" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/filesystem" + gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/security" + "github.com/portainer/portainer/api/stacks" ) type composeStackFromFileContentPayload struct { @@ -100,7 +103,6 @@ func (handler *Handler) createComposeStackFromFileContent(w http.ResponseWriter, type composeStackFromGitRepositoryPayload struct { // Name of the stack Name string `example:"myStack" validate:"required"` - // URL of a Git repository hosting the Stack file RepositoryURL string `example:"https://github.com/openfaas/faas" validate:"required"` // Reference name of a Git repository hosting the Stack file @@ -112,8 +114,10 @@ type composeStackFromGitRepositoryPayload struct { // Password used in basic authentication. Required when RepositoryAuthentication is true. RepositoryPassword string `example:"myGitPassword"` // Path to the Stack file inside the Git repository - ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` - + ComposeFile string `example:"docker-compose.yml" default:"docker-compose.yml"` + // Applicable when deploying with multiple stack files + AdditionalFiles []string + AutoUpdate *portainer.StackAutoUpdate // A list of environment variables used during stack deployment Env []portainer.Pair } @@ -122,14 +126,18 @@ func (payload *composeStackFromGitRepositoryPayload) Validate(r *http.Request) e if govalidator.IsNull(payload.Name) { return errors.New("Invalid stack name") } - if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) { return errors.New("Invalid repository URL. Must correspond to a valid URL format") } - if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { - return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") + if govalidator.IsNull(payload.RepositoryReferenceName) { + payload.RepositoryReferenceName = defaultGitReferenceName + } + if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) { + return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled") + } + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err } - return nil } @@ -141,42 +149,72 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite } payload.Name = handler.ComposeStackManager.NormalizeStackName(payload.Name) - if payload.ComposeFilePathInRepository == "" { - payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName + if payload.ComposeFile == "" { + payload.ComposeFile = filesystem.ComposeFileDefaultName } isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, false) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } if !isUnique { - errorMessage := fmt.Sprintf("A stack with the name '%s' already exists", payload.Name) - return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)} + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists} + } + + //make sure the webhook ID is unique + if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" { + isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists} + } } stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackID), - Name: payload.Name, - Type: portainer.DockerComposeStack, - EndpointID: endpoint.ID, - EntryPoint: payload.ComposeFilePathInRepository, - Env: payload.Env, + ID: portainer.StackID(stackID), + Name: payload.Name, + Type: portainer.DockerComposeStack, + EndpointID: endpoint.ID, + EntryPoint: payload.ComposeFile, + AdditionalFiles: payload.AdditionalFiles, + AutoUpdate: payload.AutoUpdate, + Env: payload.Env, + GitConfig: &gittypes.RepoConfig{ + URL: payload.RepositoryURL, + ReferenceName: payload.RepositoryReferenceName, + ConfigFilePath: payload.ComposeFile, + }, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), } + if payload.RepositoryAuthentication { + stack.GitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, + } + } + projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID))) stack.ProjectPath = projectPath doCleanUp := true defer handler.cleanUp(stack, &doCleanUp) - err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + err = handler.clone(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} } + commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err} + } + stack.GitConfig.ConfigHash = commitId + config, configErr := handler.createComposeDeployConfig(r, stack, endpoint) if configErr != nil { return configErr @@ -187,6 +225,20 @@ func (handler *Handler) createComposeStackFromGitRepository(w http.ResponseWrite return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} } + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + d, err := time.ParseDuration(payload.AutoUpdate.Interval) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Unable to parse auto update interval", Err: err} + } + jobID := handler.Scheduler.StartJobEvery(d, func() { + if err := stacks.RedeployWhenChanged(stack.ID, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil { + log.Printf("[ERROR] %s\n", err) + } + }) + + stack.AutoUpdate.JobID = jobID + } + stack.CreatedBy = config.user.Username err = handler.DataStore.Stack().CreateStack(stack) @@ -331,7 +383,7 @@ func (handler *Handler) createComposeDeployConfig(r *http.Request, stack *portai func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig) error { isAdminOrEndpointAdmin, err := handler.userIsAdminOrEndpointAdmin(config.user, config.endpoint.ID) if err != nil { - return err + return errors.Wrap(err, "failed to check user priviliges deploying a stack") } securitySettings := &config.endpoint.SecuritySettings @@ -344,15 +396,17 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig) !securitySettings.AllowContainerCapabilitiesForRegularUsers) && !isAdminOrEndpointAdmin { - composeFilePath := path.Join(config.stack.ProjectPath, config.stack.EntryPoint) - stackContent, err := handler.FileService.GetFileContent(composeFilePath) - if err != nil { - return err - } + for _, file := range append([]string{config.stack.EntryPoint}, config.stack.AdditionalFiles...) { + path := path.Join(config.stack.ProjectPath, file) + stackContent, err := handler.FileService.GetFileContent(path) + if err != nil { + return errors.Wrapf(err, "failed to get stack file content `%q`", path) + } - err = handler.isValidStackFile(stackContent, securitySettings) - if err != nil { - return err + err = handler.isValidStackFile(stackContent, securitySettings) + if err != nil { + return errors.Wrap(err, "compose file is invalid") + } } } @@ -363,7 +417,7 @@ func (handler *Handler) deployComposeStack(config *composeStackDeploymentConfig) err = handler.ComposeStackManager.Up(config.stack, config.endpoint) if err != nil { - return err + return errors.Wrap(err, "failed to start up the stack") } return handler.SwarmStackManager.Logout(config.endpoint) diff --git a/api/http/handler/stacks/create_kubernetes_stack_test.go b/api/http/handler/stacks/create_kubernetes_stack_test.go index f1b47286e..2bcd35ab5 100644 --- a/api/http/handler/stacks/create_kubernetes_stack_test.go +++ b/api/http/handler/stacks/create_kubernetes_stack_test.go @@ -23,6 +23,10 @@ func (g *git) ClonePrivateRepositoryWithBasicAuth(repositoryURL, referenceName s return g.ClonePublicRepository(repositoryURL, referenceName, destination) } +func (g *git) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { + return "", nil +} + func TestCloneAndConvertGitRepoFile(t *testing.T) { dir, err := os.MkdirTemp("", "kube-create-stack") assert.NoError(t, err, "failed to create a tmp dir") diff --git a/api/http/handler/stacks/create_swarm_stack.go b/api/http/handler/stacks/create_swarm_stack.go index b439addbf..812c22087 100644 --- a/api/http/handler/stacks/create_swarm_stack.go +++ b/api/http/handler/stacks/create_swarm_stack.go @@ -3,6 +3,7 @@ package stacks import ( "errors" "fmt" + "log" "net/http" "path" "strconv" @@ -13,7 +14,9 @@ import ( "github.com/portainer/libhttp/request" portainer "github.com/portainer/portainer/api" "github.com/portainer/portainer/api/filesystem" + gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/security" + "github.com/portainer/portainer/api/stacks" ) type swarmStackFromFileContentPayload struct { @@ -119,7 +122,9 @@ type swarmStackFromGitRepositoryPayload struct { // Password used in basic authentication. Required when RepositoryAuthentication is true. RepositoryPassword string `example:"myGitPassword"` // Path to the Stack file inside the Git repository - ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` + ComposeFile string `example:"docker-compose.yml" default:"docker-compose.yml"` + AdditionalFiles []string + AutoUpdate *portainer.StackAutoUpdate } func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) error { @@ -132,11 +137,14 @@ func (payload *swarmStackFromGitRepositoryPayload) Validate(r *http.Request) err if govalidator.IsNull(payload.RepositoryURL) || !govalidator.IsURL(payload.RepositoryURL) { return errors.New("Invalid repository URL. Must correspond to a valid URL format") } - if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { - return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") + if govalidator.IsNull(payload.RepositoryReferenceName) { + payload.RepositoryReferenceName = defaultGitReferenceName } - if govalidator.IsNull(payload.ComposeFilePathInRepository) { - payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName + if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) { + return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled") + } + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err } return nil } @@ -145,42 +153,72 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, var payload swarmStackFromGitRepositoryPayload err := request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} } isUnique, err := handler.checkUniqueName(endpoint, payload.Name, 0, true) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to check for name collision", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for name collision", Err: err} } if !isUnique { - errorMessage := fmt.Sprintf("A stack with the name '%s' is already running", payload.Name) - return &httperror.HandlerError{http.StatusConflict, errorMessage, errors.New(errorMessage)} + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.Name), Err: errStackAlreadyExists} + } + + //make sure the webhook ID is unique + if payload.AutoUpdate != nil && payload.AutoUpdate.Webhook != "" { + isUnique, err := handler.checkUniqueWebhookID(payload.AutoUpdate.Webhook) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to check for webhook ID collision", Err: err} + } + if !isUnique { + return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("Webhook ID: %s already exists", payload.AutoUpdate.Webhook), Err: errWebhookIDAlreadyExists} + } } stackID := handler.DataStore.Stack().GetNextIdentifier() stack := &portainer.Stack{ - ID: portainer.StackID(stackID), - Name: payload.Name, - Type: portainer.DockerSwarmStack, - SwarmID: payload.SwarmID, - EndpointID: endpoint.ID, - EntryPoint: payload.ComposeFilePathInRepository, + ID: portainer.StackID(stackID), + Name: payload.Name, + Type: portainer.DockerSwarmStack, + SwarmID: payload.SwarmID, + EndpointID: endpoint.ID, + EntryPoint: payload.ComposeFile, + AdditionalFiles: payload.AdditionalFiles, + AutoUpdate: payload.AutoUpdate, + GitConfig: &gittypes.RepoConfig{ + URL: payload.RepositoryURL, + ReferenceName: payload.RepositoryReferenceName, + ConfigFilePath: payload.ComposeFile, + }, Env: payload.Env, Status: portainer.StackStatusActive, CreationDate: time.Now().Unix(), } + if payload.RepositoryAuthentication { + stack.GitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, + } + } + projectPath := handler.FileService.GetStackProjectPath(strconv.Itoa(int(stack.ID))) stack.ProjectPath = projectPath doCleanUp := true defer handler.cleanUp(stack, &doCleanUp) - err = handler.cloneAndSaveConfig(stack, projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.ComposeFilePathInRepository, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + err = handler.clone(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) if err != nil { return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} } + commitId, err := handler.latestCommitID(payload.RepositoryURL, payload.RepositoryReferenceName, payload.RepositoryAuthentication, payload.RepositoryUsername, payload.RepositoryPassword) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to fetch git repository id", Err: err} + } + stack.GitConfig.ConfigHash = commitId + config, configErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) if configErr != nil { return configErr @@ -188,14 +226,28 @@ func (handler *Handler) createSwarmStackFromGitRepository(w http.ResponseWriter, err = handler.deploySwarmStack(config) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} + } + + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + d, err := time.ParseDuration(payload.AutoUpdate.Interval) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Unable to parse auto update interval", Err: err} + } + jobID := handler.Scheduler.StartJobEvery(d, func() { + if err := stacks.RedeployWhenChanged(stack.ID, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil { + log.Printf("[ERROR] %s\n", err) + } + }) + + stack.AutoUpdate.JobID = jobID } stack.CreatedBy = config.user.Username err = handler.DataStore.Stack().CreateStack(stack) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack inside the database", Err: err} } doCleanUp = false @@ -344,16 +396,17 @@ func (handler *Handler) deploySwarmStack(config *swarmStackDeploymentConfig) err settings := &config.endpoint.SecuritySettings if !settings.AllowBindMountsForRegularUsers && !isAdminOrEndpointAdmin { - composeFilePath := path.Join(config.stack.ProjectPath, config.stack.EntryPoint) + for _, file := range append([]string{config.stack.EntryPoint}, config.stack.AdditionalFiles...) { + path := path.Join(config.stack.ProjectPath, file) + stackContent, err := handler.FileService.GetFileContent(path) + if err != nil { + return err + } - stackContent, err := handler.FileService.GetFileContent(composeFilePath) - if err != nil { - return err - } - - err = handler.isValidStackFile(stackContent, settings) - if err != nil { - return err + err = handler.isValidStackFile(stackContent, settings) + if err != nil { + return err + } } } diff --git a/api/http/handler/stacks/handler.go b/api/http/handler/stacks/handler.go index dcdfce41b..7cb2558e8 100644 --- a/api/http/handler/stacks/handler.go +++ b/api/http/handler/stacks/handler.go @@ -2,23 +2,30 @@ package stacks import ( "context" - "errors" + "fmt" "net/http" "strings" "sync" "github.com/docker/docker/api/types" "github.com/gorilla/mux" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" portainer "github.com/portainer/portainer/api" + bolterrors "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/docker" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/authorization" + "github.com/portainer/portainer/api/scheduler" + "github.com/portainer/portainer/api/stacks" ) +const defaultGitReferenceName = "refs/heads/master" + var ( - errStackAlreadyExists = errors.New("A stack already exists with this name") - errStackNotExternal = errors.New("Not an external stack") + errStackAlreadyExists = errors.New("A stack already exists with this name") + errWebhookIDAlreadyExists = errors.New("A webhook ID already exists") + errStackNotExternal = errors.New("Not an external stack") ) // Handler is the HTTP handler used to handle stack operations. @@ -34,6 +41,8 @@ type Handler struct { SwarmStackManager portainer.SwarmStackManager ComposeStackManager portainer.ComposeStackManager KubernetesDeployer portainer.KubernetesDeployer + Scheduler *scheduler.Scheduler + StackDeployer stacks.StackDeployer } // NewHandler creates a handler to manage stack operations. @@ -57,7 +66,9 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { h.Handle("/stacks/{id}", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdate))).Methods(http.MethodPut) h.Handle("/stacks/{id}/git", - bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackUpdateGit))).Methods(http.MethodPut) + bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackGitUpdate))).Methods(http.MethodPost) + h.Handle("/stacks/{id}/git/redeploy", + bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackGitRedeploy))).Methods(http.MethodPut) h.Handle("/stacks/{id}/file", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackFile))).Methods(http.MethodGet) h.Handle("/stacks/{id}/migrate", @@ -66,6 +77,9 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackStart))).Methods(http.MethodPost) h.Handle("/stacks/{id}/stop", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.stackStop))).Methods(http.MethodPost) + h.Handle("/stacks/webhook/{webhookID}", + httperror.LoggerHandler(h.webhookInvoke)).Methods(http.MethodPost) + return h } @@ -159,3 +173,34 @@ func (handler *Handler) checkUniqueName(endpoint *portainer.Endpoint, name strin return true, nil } + +func (handler *Handler) checkUniqueWebhookID(webhookID string) (bool, error) { + _, err := handler.DataStore.Stack().StackByWebhookID(webhookID) + if err == bolterrors.ErrObjectNotFound { + return true, nil + } + return false, err +} + +func (handler *Handler) clone(projectPath, repositoryURL, refName string, auth bool, username, password string) error { + if !auth { + username = "" + password = "" + } + + err := handler.GitService.CloneRepository(projectPath, repositoryURL, refName, username, password) + if err != nil { + return fmt.Errorf("unable to clone git repository: %w", err) + } + + return nil +} + +func (handler *Handler) latestCommitID(repositoryURL, refName string, auth bool, username, password string) (string, error) { + if !auth { + username = "" + password = "" + } + + return handler.GitService.LatestCommitID(repositoryURL, refName, username, password) +} diff --git a/api/http/handler/stacks/helper.go b/api/http/handler/stacks/helper.go new file mode 100644 index 000000000..dd42330c4 --- /dev/null +++ b/api/http/handler/stacks/helper.go @@ -0,0 +1,24 @@ +package stacks + +import ( + "time" + + "github.com/asaskevich/govalidator" + "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" +) + +func validateStackAutoUpdate(autoUpdate *portainer.StackAutoUpdate) error { + if autoUpdate == nil { + return nil + } + if autoUpdate.Webhook != "" && !govalidator.IsUUID(autoUpdate.Webhook) { + return errors.New("invalid Webhook format") + } + if autoUpdate.Interval != "" { + if _, err := time.ParseDuration(autoUpdate.Interval); err != nil { + return errors.New("invalid Interval format") + } + } + return nil +} diff --git a/api/http/handler/stacks/helper_test.go b/api/http/handler/stacks/helper_test.go new file mode 100644 index 000000000..c3e564349 --- /dev/null +++ b/api/http/handler/stacks/helper_test.go @@ -0,0 +1,42 @@ +package stacks + +import ( + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/stretchr/testify/assert" +) + +func Test_ValidateStackAutoUpdate(t *testing.T) { + tests := []struct { + name string + value *portainer.StackAutoUpdate + wantErr bool + }{ + { + name: "webhook is not a valid UUID", + value: &portainer.StackAutoUpdate{Webhook: "fake-webhook"}, + wantErr: true, + }, + { + name: "incorrect interval value", + value: &portainer.StackAutoUpdate{Interval: "1dd2hh3mm"}, + wantErr: true, + }, + { + name: "valid auto update", + value: &portainer.StackAutoUpdate{ + Webhook: "8dce8c2f-9ca1-482b-ad20-271e86536ada", + Interval: "5h30m40s10ms", + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateStackAutoUpdate(tt.value) + assert.Equalf(t, tt.wantErr, err != nil, "received %+v", err) + }) + } +} diff --git a/api/http/handler/stacks/stack_create.go b/api/http/handler/stacks/stack_create.go index 53533ca33..0f3cbb437 100644 --- a/api/http/handler/stacks/stack_create.go +++ b/api/http/handler/stacks/stack_create.go @@ -1,19 +1,17 @@ package stacks import ( - "errors" - "fmt" "log" "net/http" "github.com/docker/cli/cli/compose/loader" "github.com/docker/cli/cli/compose/types" + "github.com/pkg/errors" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" bolterrors "github.com/portainer/portainer/api/bolt/errors" - gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/authorization" "github.com/portainer/portainer/api/internal/endpointutils" @@ -129,7 +127,7 @@ func (handler *Handler) createComposeStack(w http.ResponseWriter, r *http.Reques return handler.createComposeStackFromFileUpload(w, r, endpoint, userID) } - return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: method. Value must be one of: string, repository or file", errors.New(request.ErrInvalidQueryParameter)} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid value for query parameter: method. Value must be one of: string, repository or file", Err: errors.New(request.ErrInvalidQueryParameter)} } func (handler *Handler) createSwarmStack(w http.ResponseWriter, r *http.Request, method string, endpoint *portainer.Endpoint, userID portainer.UserID) *httperror.HandlerError { @@ -142,7 +140,7 @@ func (handler *Handler) createSwarmStack(w http.ResponseWriter, r *http.Request, return handler.createSwarmStackFromFileUpload(w, r, endpoint, userID) } - return &httperror.HandlerError{http.StatusBadRequest, "Invalid value for query parameter: method. Value must be one of: string, repository or file", errors.New(request.ErrInvalidQueryParameter)} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid value for query parameter: method. Value must be one of: string, repository or file", Err: errors.New(request.ErrInvalidQueryParameter)} } func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Request, method string, endpoint *portainer.Endpoint) *httperror.HandlerError { @@ -234,22 +232,3 @@ func (handler *Handler) decorateStackResponse(w http.ResponseWriter, stack *port stack.ResourceControl = resourceControl return response.JSON(w, stack) } - -func (handler *Handler) cloneAndSaveConfig(stack *portainer.Stack, projectPath, repositoryURL, refName, configFilePath string, auth bool, username, password string) error { - if !auth { - username = "" - password = "" - } - - err := handler.GitService.CloneRepository(projectPath, repositoryURL, refName, username, password) - if err != nil { - return fmt.Errorf("unable to clone git repository: %w", err) - } - - stack.GitConfig = &gittypes.RepoConfig{ - URL: repositoryURL, - ReferenceName: refName, - ConfigFilePath: configFilePath, - } - return nil -} diff --git a/api/http/handler/stacks/stack_create_test.go b/api/http/handler/stacks/stack_create_test.go deleted file mode 100644 index 414948378..000000000 --- a/api/http/handler/stacks/stack_create_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package stacks - -import ( - "testing" - - portainer "github.com/portainer/portainer/api" - gittypes "github.com/portainer/portainer/api/git/types" - "github.com/portainer/portainer/api/http/security" - "github.com/portainer/portainer/api/internal/testhelpers" - "github.com/stretchr/testify/assert" -) - -func Test_stackHandler_cloneAndSaveConfig_shouldCallGitCloneAndSaveConfigOnStack(t *testing.T) { - handler := NewHandler(&security.RequestBouncer{}) - handler.GitService = testhelpers.NewGitService() - - url := "url" - refName := "ref" - configPath := "path" - stack := &portainer.Stack{} - err := handler.cloneAndSaveConfig(stack, "", url, refName, configPath, false, "", "") - assert.NoError(t, err, "clone and save should not fail") - - assert.Equal(t, gittypes.RepoConfig{ - URL: url, - ReferenceName: refName, - ConfigFilePath: configPath, - }, *stack.GitConfig) -} diff --git a/api/http/handler/stacks/stack_delete.go b/api/http/handler/stacks/stack_delete.go index 52f003cea..aad4ff197 100644 --- a/api/http/handler/stacks/stack_delete.go +++ b/api/http/handler/stacks/stack_delete.go @@ -96,6 +96,11 @@ func (handler *Handler) stackDelete(w http.ResponseWriter, r *http.Request) *htt } } + // stop scheduler updates of the stack before removal + if stack.AutoUpdate != nil && stack.AutoUpdate.JobID != "" { + handler.Scheduler.StopJob(stack.AutoUpdate.JobID) + } + err = handler.deleteStack(stack, endpoint) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} diff --git a/api/http/handler/stacks/stack_start.go b/api/http/handler/stacks/stack_start.go index 0a915600b..955a329a6 100644 --- a/api/http/handler/stacks/stack_start.go +++ b/api/http/handler/stacks/stack_start.go @@ -3,12 +3,15 @@ package stacks import ( "errors" "fmt" + "log" "net/http" + "time" portainer "github.com/portainer/portainer/api" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/stackutils" + "github.com/portainer/portainer/api/stacks" httperror "github.com/portainer/libhttp/error" "github.com/portainer/libhttp/request" @@ -85,6 +88,26 @@ func (handler *Handler) stackStart(w http.ResponseWriter, r *http.Request) *http return &httperror.HandlerError{http.StatusBadRequest, "Stack is already active", errors.New("Stack is already active")} } + if stack.AutoUpdate != nil && stack.AutoUpdate.Interval != "" { + if stack.AutoUpdate.JobID != "" { + if err := handler.Scheduler.StopJob(stack.AutoUpdate.JobID); err != nil { + log.Printf("[WARN] could not stop the job for the stack %v", stack.ID) + } + } + + d, err := time.ParseDuration(stack.AutoUpdate.Interval) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Unable to parse stack's auto update interval", Err: err} + } + jobID := handler.Scheduler.StartJobEvery(d, func() { + if err := stacks.RedeployWhenChanged(stack.ID, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil { + log.Printf("[ERROR] %s\n", err) + } + }) + + stack.AutoUpdate.JobID = jobID + } + err = handler.startStack(stack, endpoint) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to start stack", err} diff --git a/api/http/handler/stacks/stack_stop.go b/api/http/handler/stacks/stack_stop.go index 7dabfb265..ce13798a1 100644 --- a/api/http/handler/stacks/stack_stop.go +++ b/api/http/handler/stacks/stack_stop.go @@ -74,6 +74,12 @@ func (handler *Handler) stackStop(w http.ResponseWriter, r *http.Request) *httpe return &httperror.HandlerError{http.StatusBadRequest, "Stack is already inactive", errors.New("Stack is already inactive")} } + // stop scheduler updates of the stack before stopping + if stack.AutoUpdate != nil && stack.AutoUpdate.JobID != "" { + handler.Scheduler.StopJob(stack.AutoUpdate.JobID) + stack.AutoUpdate.JobID = "" + } + err = handler.stopStack(stack, endpoint) if err != nil { return &httperror.HandlerError{http.StatusInternalServerError, "Unable to stop stack", err} diff --git a/api/http/handler/stacks/stack_update_git.go b/api/http/handler/stacks/stack_update_git.go index 9d266cd89..3f0c591af 100644 --- a/api/http/handler/stacks/stack_update_git.go +++ b/api/http/handler/stacks/stack_update_git.go @@ -2,7 +2,6 @@ package stacks import ( "errors" - "fmt" "log" "net/http" "time" @@ -13,42 +12,74 @@ import ( "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" bolterrors "github.com/portainer/portainer/api/bolt/errors" - "github.com/portainer/portainer/api/filesystem" + gittypes "github.com/portainer/portainer/api/git/types" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/stackutils" + "github.com/portainer/portainer/api/stacks" ) -type updateStackGitPayload struct { +type stackGitUpdatePayload struct { + AutoUpdate *portainer.StackAutoUpdate + Env []portainer.Pair RepositoryReferenceName string RepositoryAuthentication bool RepositoryUsername string RepositoryPassword string } -func (payload *updateStackGitPayload) Validate(r *http.Request) error { - if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { - return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") +func (payload *stackGitUpdatePayload) Validate(r *http.Request) error { + if govalidator.IsNull(payload.RepositoryReferenceName) { + return errors.New("Invalid RepositoryReferenceName") + } + if govalidator.IsNull(payload.RepositoryReferenceName) { + payload.RepositoryReferenceName = defaultGitReferenceName + } + if payload.RepositoryAuthentication && govalidator.IsNull(payload.RepositoryPassword) { + return errors.New("Invalid repository credentials. Password must be specified when authentication is enabled") + } + if err := validateStackAutoUpdate(payload.AutoUpdate); err != nil { + return err } return nil } -// PUT request on /api/stacks/:id/git?endpointId= -func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { +// @id Stacks +// @summary Update and redeploy an existing stack (with Git config) +// @description Update and redeploy an existing stack (with Git config) +// @description **Access policy**: authenticated +// @tags stacks +// @security jwt +// @produce json +// @param id path int true "Stack identifier" +// @param endpointId query int false "Stacks created before version 1.18.0 might not have an associated endpoint identifier. Use this optional parameter to set the endpoint identifier used by the stack." +// @param body body stackGitUpdatePayload true "Stack Git config" +// @success 200 {object} portainer.Stack "Success" +// @failure 400 "Invalid request" +// @failure 403 "Permission denied" +// @failure 404 "Not found" +// @failure 500 "Server error" +// @router /stacks/{id}/git +func (handler *Handler) stackGitUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} + } + + var payload stackGitUpdatePayload + err = request.DecodeAndValidateJSONPayload(r, &payload) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} } stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID)) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a stack with the specified identifier inside the database", err} - } - - if stack.GitConfig == nil { - return &httperror.HandlerError{http.StatusBadRequest, "Stack is not created from git", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } else if stack.GitConfig == nil { + msg := "No Git config in the found stack" + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: msg, Err: errors.New(msg)} } // TODO: this is a work-around for stacks created with Portainer version >= 1.17.1 @@ -56,7 +87,7 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * // can use the optional EndpointID query parameter to associate a valid endpoint identifier to the stack. endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true) if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid query parameter: endpointId", err} + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} } if endpointID != int(stack.EndpointID) { stack.EndpointID = portainer.EndpointID(endpointID) @@ -64,117 +95,75 @@ func (handler *Handler) stackUpdateGit(w http.ResponseWriter, r *http.Request) * endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) if err == bolterrors.ErrObjectNotFound { - return &httperror.HandlerError{http.StatusNotFound, "Unable to find the endpoint associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } else if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find the endpoint associated to the stack inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} } err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) if err != nil { - return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} } resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a resource control associated to the stack", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} } securityContext, err := security.RetrieveRestrictedRequestContext(r) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve info from request context", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} } access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to verify user authorizations to validate stack access", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} } if !access { - return &httperror.HandlerError{http.StatusForbidden, "Access denied to resource", httperrors.ErrResourceAccessDenied} + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} } - var payload updateStackGitPayload - err = request.DecodeAndValidateJSONPayload(r, &payload) - if err != nil { - return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err} + //stop the autoupdate job if there is any + if stack.AutoUpdate != nil && stack.AutoUpdate.JobID != "" { + handler.Scheduler.StopJob(stack.AutoUpdate.JobID) } + //update retrieved stack data based on the payload stack.GitConfig.ReferenceName = payload.RepositoryReferenceName + stack.AutoUpdate = payload.AutoUpdate + stack.Env = payload.Env - backupProjectPath := fmt.Sprintf("%s-old", stack.ProjectPath) - err = filesystem.MoveDirectory(stack.ProjectPath, backupProjectPath) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to move git repository directory", err} - } - - repositoryUsername := payload.RepositoryUsername - repositoryPassword := payload.RepositoryPassword - if !payload.RepositoryAuthentication { - repositoryUsername = "" - repositoryPassword = "" - } - - err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword) - if err != nil { - restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath) - if restoreError != nil { - log.Printf("[WARN] [http,stacks,git] [error: %s] [message: failed restoring backup folder]", restoreError) + if payload.RepositoryAuthentication { + stack.GitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, + } + } else { + stack.GitConfig.Authentication = &gittypes.GitAuthentication{ + Username: "", + Password: "", } - - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to clone git repository", err} } - defer func() { - err = handler.FileService.RemoveDirectory(backupProjectPath) + if payload.AutoUpdate != nil && payload.AutoUpdate.Interval != "" { + d, err := time.ParseDuration(payload.AutoUpdate.Interval) if err != nil { - log.Printf("[WARN] [http,stacks,git] [error: %s] [message: unable to remove git repository directory]", err) + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Unable to parse auto update interval", Err: err} } - }() + jobID := handler.Scheduler.StartJobEvery(d, func() { + if err := stacks.RedeployWhenChanged(stack.ID, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil { + log.Printf("[ERROR] %s\n", err) + } + }) - httpErr := handler.deployStack(r, stack, endpoint) - if httpErr != nil { - return httpErr + stack.AutoUpdate.JobID = jobID } + //save the updated stack to DB err = handler.DataStore.Stack().UpdateStack(stack.ID, stack) if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err} + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err} } return response.JSON(w, stack) } - -func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError { - if stack.Type == portainer.DockerSwarmStack { - config, httpErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) - if httpErr != nil { - return httpErr - } - - err := handler.deploySwarmStack(config) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} - } - - stack.UpdateDate = time.Now().Unix() - stack.UpdatedBy = config.user.Username - stack.Status = portainer.StackStatusActive - - return nil - } - - config, httpErr := handler.createComposeDeployConfig(r, stack, endpoint) - if httpErr != nil { - return httpErr - } - - err := handler.deployComposeStack(config) - if err != nil { - return &httperror.HandlerError{http.StatusInternalServerError, err.Error(), err} - } - - stack.UpdateDate = time.Now().Unix() - stack.UpdatedBy = config.user.Username - stack.Status = portainer.StackStatusActive - - return nil -} diff --git a/api/http/handler/stacks/stack_update_git_redeploy.go b/api/http/handler/stacks/stack_update_git_redeploy.go new file mode 100644 index 000000000..cae20ef5f --- /dev/null +++ b/api/http/handler/stacks/stack_update_git_redeploy.go @@ -0,0 +1,182 @@ +package stacks + +import ( + "errors" + "fmt" + "log" + "net/http" + "time" + + "github.com/asaskevich/govalidator" + httperror "github.com/portainer/libhttp/error" + "github.com/portainer/libhttp/request" + "github.com/portainer/libhttp/response" + portainer "github.com/portainer/portainer/api" + bolterrors "github.com/portainer/portainer/api/bolt/errors" + "github.com/portainer/portainer/api/filesystem" + httperrors "github.com/portainer/portainer/api/http/errors" + "github.com/portainer/portainer/api/http/security" + "github.com/portainer/portainer/api/internal/stackutils" +) + +type stackGitRedployPayload struct { + RepositoryReferenceName string + RepositoryAuthentication bool + RepositoryUsername string + RepositoryPassword string + Env []portainer.Pair +} + +func (payload *stackGitRedployPayload) Validate(r *http.Request) error { + if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { + return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") + } + return nil +} + +// PUT request on /api/stacks/:id/git?endpointId= +func (handler *Handler) stackGitRedeploy(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + stackID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid stack identifier route variable", Err: err} + } + + stack, err := handler.DataStore.Stack().Stack(portainer.StackID(stackID)) + if err == bolterrors.ErrObjectNotFound { + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } else if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find a stack with the specified identifier inside the database", Err: err} + } + + if stack.GitConfig == nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Stack is not created from git", Err: err} + } + + // TODO: this is a work-around for stacks created with Portainer version >= 1.17.1 + // The EndpointID property is not available for these stacks, this API endpoint + // can use the optional EndpointID query parameter to associate a valid endpoint identifier to the stack. + endpointID, err := request.RetrieveNumericQueryParameter(r, "endpointId", true) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid query parameter: endpointId", Err: err} + } + if endpointID != int(stack.EndpointID) { + stack.EndpointID = portainer.EndpointID(endpointID) + } + + endpoint, err := handler.DataStore.Endpoint().Endpoint(stack.EndpointID) + if err == bolterrors.ErrObjectNotFound { + return &httperror.HandlerError{StatusCode: http.StatusNotFound, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} + } else if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to find the endpoint associated to the stack inside the database", Err: err} + } + + err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Permission denied to access endpoint", Err: err} + } + + resourceControl, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(stackutils.ResourceControlID(stack.EndpointID, stack.Name), portainer.StackResourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve a resource control associated to the stack", Err: err} + } + + securityContext, err := security.RetrieveRestrictedRequestContext(r) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to retrieve info from request context", Err: err} + } + + access, err := handler.userCanAccessStack(securityContext, endpoint.ID, resourceControl) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to verify user authorizations to validate stack access", Err: err} + } + if !access { + return &httperror.HandlerError{StatusCode: http.StatusForbidden, Message: "Access denied to resource", Err: httperrors.ErrResourceAccessDenied} + } + + var payload stackGitRedployPayload + err = request.DecodeAndValidateJSONPayload(r, &payload) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid request payload", Err: err} + } + + stack.GitConfig.ReferenceName = payload.RepositoryReferenceName + stack.Env = payload.Env + + backupProjectPath := fmt.Sprintf("%s-old", stack.ProjectPath) + err = filesystem.MoveDirectory(stack.ProjectPath, backupProjectPath) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to move git repository directory", Err: err} + } + + repositoryUsername := payload.RepositoryUsername + repositoryPassword := payload.RepositoryPassword + if !payload.RepositoryAuthentication { + repositoryUsername = "" + repositoryPassword = "" + } + + err = handler.GitService.CloneRepository(stack.ProjectPath, stack.GitConfig.URL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword) + if err != nil { + restoreError := filesystem.MoveDirectory(backupProjectPath, stack.ProjectPath) + if restoreError != nil { + log.Printf("[WARN] [http,stacks,git] [error: %s] [message: failed restoring backup folder]", restoreError) + } + + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to clone git repository", Err: err} + } + + defer func() { + err = handler.FileService.RemoveDirectory(backupProjectPath) + if err != nil { + log.Printf("[WARN] [http,stacks,git] [error: %s] [message: unable to remove git repository directory]", err) + } + }() + + httpErr := handler.deployStack(r, stack, endpoint) + if httpErr != nil { + return httpErr + } + + err = handler.DataStore.Stack().UpdateStack(stack.ID, stack) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Unable to persist the stack changes inside the database", Err: err} + } + + return response.JSON(w, stack) +} + +func (handler *Handler) deployStack(r *http.Request, stack *portainer.Stack, endpoint *portainer.Endpoint) *httperror.HandlerError { + if stack.Type == portainer.DockerSwarmStack { + config, httpErr := handler.createSwarmDeployConfig(r, stack, endpoint, false) + if httpErr != nil { + return httpErr + } + + err := handler.deploySwarmStack(config) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} + } + + stack.UpdateDate = time.Now().Unix() + stack.UpdatedBy = config.user.Username + stack.Status = portainer.StackStatusActive + + return nil + } + + config, httpErr := handler.createComposeDeployConfig(r, stack, endpoint) + if httpErr != nil { + return httpErr + } + + err := handler.deployComposeStack(config) + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: err.Error(), Err: err} + } + + stack.UpdateDate = time.Now().Unix() + stack.UpdatedBy = config.user.Username + stack.Status = portainer.StackStatusActive + + return nil +} diff --git a/api/http/handler/stacks/webhook_invoke.go b/api/http/handler/stacks/webhook_invoke.go new file mode 100644 index 000000000..01c9d701b --- /dev/null +++ b/api/http/handler/stacks/webhook_invoke.go @@ -0,0 +1,54 @@ +package stacks + +import ( + "log" + "net/http" + + "github.com/gofrs/uuid" + + "github.com/portainer/libhttp/response" + + bolterrors "github.com/portainer/portainer/api/bolt/errors" + "github.com/portainer/portainer/api/stacks" + + httperror "github.com/portainer/libhttp/error" + "github.com/portainer/libhttp/request" +) + +func (handler *Handler) webhookInvoke(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + webhookID, err := retrieveUUIDRouteVariableValue(r, "webhookID") + if err != nil { + return &httperror.HandlerError{StatusCode: http.StatusBadRequest, Message: "Invalid webhook identifier route variable", Err: err} + } + + stack, err := handler.DataStore.Stack().StackByWebhookID(webhookID.String()) + if err != nil { + statusCode := http.StatusInternalServerError + if err == bolterrors.ErrObjectNotFound { + statusCode = http.StatusNotFound + } + return &httperror.HandlerError{StatusCode: statusCode, Message: "Unable to find the stack by webhook ID", Err: err} + } + + if err = stacks.RedeployWhenChanged(stack.ID, handler.StackDeployer, handler.DataStore, handler.GitService); err != nil { + log.Printf("[ERROR] %s\n", err) + return &httperror.HandlerError{StatusCode: http.StatusInternalServerError, Message: "Failed to update the stack", Err: err} + } + + return response.Empty(w) +} + +func retrieveUUIDRouteVariableValue(r *http.Request, name string) (uuid.UUID, error) { + webhookID, err := request.RetrieveRouteVariableValue(r, name) + if err != nil { + return uuid.Nil, err + } + + uid, err := uuid.FromString(webhookID) + + if err != nil { + return uuid.Nil, err + } + + return uid, nil +} diff --git a/api/http/handler/stacks/webhook_invoke_test.go b/api/http/handler/stacks/webhook_invoke_test.go new file mode 100644 index 000000000..a6aae064a --- /dev/null +++ b/api/http/handler/stacks/webhook_invoke_test.go @@ -0,0 +1,59 @@ +package stacks + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gofrs/uuid" + "github.com/stretchr/testify/assert" + + portainer "github.com/portainer/portainer/api" + + "github.com/portainer/portainer/api/bolt/bolttest" +) + +func TestHandler_webhookInvoke(t *testing.T) { + store, teardown := bolttest.MustNewTestStore(true) + defer teardown() + + webhookID := newGuidString(t) + store.StackService.CreateStack(&portainer.Stack{ + AutoUpdate: &portainer.StackAutoUpdate{ + Webhook: webhookID, + }, + }) + + h := NewHandler(nil) + h.DataStore = store + + t.Run("invalid uuid results in http.StatusBadRequest", func(t *testing.T) { + w := httptest.NewRecorder() + req := newRequest("notuuid") + h.Router.ServeHTTP(w, req) + assert.Equal(t, http.StatusBadRequest, w.Code) + }) + t.Run("registered webhook ID in http.StatusNoContent", func(t *testing.T) { + w := httptest.NewRecorder() + req := newRequest(webhookID) + h.Router.ServeHTTP(w, req) + assert.Equal(t, http.StatusNoContent, w.Code) + }) + t.Run("unregistered webhook ID in http.StatusNotFound", func(t *testing.T) { + w := httptest.NewRecorder() + req := newRequest(newGuidString(t)) + h.Router.ServeHTTP(w, req) + assert.Equal(t, http.StatusNotFound, w.Code) + }) +} + +func newGuidString(t *testing.T) string { + uuid, err := uuid.NewV4() + assert.NoError(t, err) + + return uuid.String() +} + +func newRequest(webhookID string) *http.Request { + return httptest.NewRequest(http.MethodPost, "/stacks/webhook/"+webhookID, nil) +} diff --git a/api/http/server.go b/api/http/server.go index ac5fe0e1b..52b109718 100644 --- a/api/http/server.go +++ b/api/http/server.go @@ -46,11 +46,13 @@ import ( "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/authorization" "github.com/portainer/portainer/api/kubernetes/cli" + "github.com/portainer/portainer/api/scheduler" + stackdeployer "github.com/portainer/portainer/api/stacks" ) // Server implements the portainer.Server interface type Server struct { - AuthorizationService *authorization.Service + AuthorizationService *authorization.Service BindAddress string AssetsPath string Status *portainer.Status @@ -75,8 +77,10 @@ type Server struct { DockerClientFactory *docker.ClientFactory KubernetesClientFactory *cli.ClientFactory KubernetesDeployer portainer.KubernetesDeployer + Scheduler *scheduler.Scheduler ShutdownCtx context.Context ShutdownTrigger context.CancelFunc + StackDeployer stackdeployer.StackDeployer } // Start starts the HTTP server @@ -174,10 +178,12 @@ func (server *Server) Start() error { stackHandler.DataStore = server.DataStore stackHandler.DockerClientFactory = server.DockerClientFactory stackHandler.FileService = server.FileService - stackHandler.SwarmStackManager = server.SwarmStackManager - stackHandler.ComposeStackManager = server.ComposeStackManager stackHandler.KubernetesDeployer = server.KubernetesDeployer stackHandler.GitService = server.GitService + stackHandler.Scheduler = server.Scheduler + stackHandler.SwarmStackManager = server.SwarmStackManager + stackHandler.ComposeStackManager = server.ComposeStackManager + stackHandler.StackDeployer = server.StackDeployer var tagHandler = tags.NewHandler(requestBouncer) tagHandler.DataStore = server.DataStore diff --git a/api/internal/stackutils/stackutils.go b/api/internal/stackutils/stackutils.go index 5b1e9bf43..7e94bff17 100644 --- a/api/internal/stackutils/stackutils.go +++ b/api/internal/stackutils/stackutils.go @@ -2,6 +2,7 @@ package stackutils import ( "fmt" + "path" portainer "github.com/portainer/portainer/api" ) @@ -10,3 +11,12 @@ import ( func ResourceControlID(endpointID portainer.EndpointID, name string) string { return fmt.Sprintf("%d_%s", endpointID, name) } + +// GetStackFilePaths returns a list of file paths based on stack project path +func GetStackFilePaths(stack *portainer.Stack) []string { + var filePaths []string + for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) { + filePaths = append(filePaths, path.Join(stack.ProjectPath, file)) + } + return filePaths +} diff --git a/api/internal/stackutils/stackutils_test.go b/api/internal/stackutils/stackutils_test.go new file mode 100644 index 000000000..6af19d8af --- /dev/null +++ b/api/internal/stackutils/stackutils_test.go @@ -0,0 +1,26 @@ +package stackutils + +import ( + "testing" + + portainer "github.com/portainer/portainer/api" + "github.com/stretchr/testify/assert" +) + +func Test_GetStackFilePaths(t *testing.T) { + stack := &portainer.Stack{ + ProjectPath: "/tmp/stack/1", + EntryPoint: "file-one.yml", + } + + t.Run("stack doesn't have additional files", func(t *testing.T) { + expected := []string{"/tmp/stack/1/file-one.yml"} + assert.ElementsMatch(t, expected, GetStackFilePaths(stack)) + }) + + t.Run("stack has additional files", func(t *testing.T) { + stack.AdditionalFiles = []string{"file-two.yml", "file-three.yml"} + expected := []string{"/tmp/stack/1/file-one.yml", "/tmp/stack/1/file-two.yml", "/tmp/stack/1/file-three.yml"} + assert.ElementsMatch(t, expected, GetStackFilePaths(stack)) + }) +} diff --git a/api/libcompose/compose_stack.go b/api/libcompose/compose_stack.go index 41da7239d..f2dfff230 100644 --- a/api/libcompose/compose_stack.go +++ b/api/libcompose/compose_stack.go @@ -16,6 +16,7 @@ import ( "github.com/portainer/libcompose/project" "github.com/portainer/libcompose/project/options" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/internal/stackutils" ) const ( @@ -86,12 +87,12 @@ func (manager *ComposeStackManager) Up(stack *portainer.Stack, endpoint *portain for _, envvar := range stack.Env { env[envvar.Name] = envvar.Value } + filePaths := stackutils.GetStackFilePaths(stack) - composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint) proj, err := docker.NewProject(&ctx.Context{ ConfigDir: manager.dataPath, Context: project.Context{ - ComposeFiles: []string{composeFilePath}, + ComposeFiles: filePaths, EnvironmentLookup: &lookup.ComposableEnvLookup{ Lookups: []config.EnvironmentLookup{ &lookup.EnvfileLookup{ @@ -120,10 +121,13 @@ func (manager *ComposeStackManager) Down(stack *portainer.Stack, endpoint *porta return err } - composeFilePath := path.Join(stack.ProjectPath, stack.EntryPoint) + var composeFiles []string + for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) { + composeFiles = append(composeFiles, path.Join(stack.ProjectPath, file)) + } proj, err := docker.NewProject(&ctx.Context{ Context: project.Context{ - ComposeFiles: []string{composeFilePath}, + ComposeFiles: composeFiles, ProjectName: stack.Name, }, ClientFactory: clientFactory, @@ -134,3 +138,11 @@ func (manager *ComposeStackManager) Down(stack *portainer.Stack, endpoint *porta return proj.Down(context.Background(), options.Down{RemoveVolume: false, RemoveOrphans: true}) } + +func stackFilePaths(stack *portainer.Stack) []string { + var filePaths []string + for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) { + filePaths = append(filePaths, path.Join(stack.ProjectPath, file)) + } + return filePaths +} diff --git a/api/portainer.go b/api/portainer.go index ae971cee2..bf6747710 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -721,10 +721,21 @@ type ( UpdateDate int64 `example:"1587399600"` // The username which last updated this stack UpdatedBy string `example:"bob"` + // Only applies when deploying stack with multiple files + AdditionalFiles []string `json:"AdditionalFiles"` + // The auto update settings of a git stack + AutoUpdate *StackAutoUpdate `json:"AutoUpdate"` // The git config of this stack GitConfig *gittypes.RepoConfig } + //StackAutoUpdate represents the git auto sync config for stack deployment + StackAutoUpdate struct { + Interval string + Webhook string //a UUID generated from client + JobID string + } + // StackID represents a stack identifier (it must be composed of Name + "_" + SwarmID to create a unique identifier) StackID int @@ -1158,6 +1169,7 @@ type ( // GitService represents a service for managing Git GitService interface { CloneRepository(destination string, repositoryURL, referenceName, username, password string) error + LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) } // JWTService represents a service for managing JWT tokens @@ -1264,6 +1276,8 @@ type ( UpdateStack(ID StackID, stack *Stack) error DeleteStack(ID StackID) error GetNextIdentifier() int + StackByWebhookID(ID string) (*Stack, error) + RefreshableStacks() ([]Stack, error) } // SnapshotService represents a service for managing endpoint snapshots diff --git a/api/scheduler/scheduler.go b/api/scheduler/scheduler.go new file mode 100644 index 000000000..6568f753f --- /dev/null +++ b/api/scheduler/scheduler.go @@ -0,0 +1,73 @@ +package scheduler + +import ( + "context" + "log" + "strconv" + "time" + + "github.com/pkg/errors" + "github.com/robfig/cron/v3" +) + +type Scheduler struct { + crontab *cron.Cron + shutdownCtx context.Context +} + +func NewScheduler(ctx context.Context) *Scheduler { + crontab := cron.New(cron.WithChain(cron.Recover(cron.DefaultLogger))) + crontab.Start() + + s := &Scheduler{ + crontab: crontab, + } + + if ctx != nil { + go func() { + <-ctx.Done() + s.Shutdown() + }() + } + + return s +} + +// Shutdown stops the scheduler and waits for it to stop if it is running; otherwise does nothing. +func (s *Scheduler) Shutdown() error { + if s.crontab == nil { + return nil + } + + log.Println("[DEBUG] Stopping scheduler") + ctx := s.crontab.Stop() + <-ctx.Done() + + for _, j := range s.crontab.Entries() { + s.crontab.Remove(j.ID) + } + + err := ctx.Err() + if err == context.Canceled { + return nil + } + return err +} + +// StopJob stops the job from being run in the future +func (s *Scheduler) StopJob(jobID string) error { + id, err := strconv.Atoi(jobID) + if err != nil { + return errors.Wrapf(err, "failed convert jobID %q to int", jobID) + } + s.crontab.Remove(cron.EntryID(id)) + + return nil +} + +// StartJobEvery schedules a new periodic job with a given duration. +// Returns job id that could be used to stop the given job +func (s *Scheduler) StartJobEvery(duration time.Duration, job func()) string { + entryId := s.crontab.Schedule(cron.Every(duration), cron.FuncJob(job)) + return strconv.Itoa(int(entryId)) +} diff --git a/api/scheduler/scheduler_test.go b/api/scheduler/scheduler_test.go new file mode 100644 index 000000000..6d21e49ec --- /dev/null +++ b/api/scheduler/scheduler_test.go @@ -0,0 +1,57 @@ +package scheduler + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func Test_CanStartAndTerminate(t *testing.T) { + s := NewScheduler(context.Background()) + s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") }) + + err := s.Shutdown() + assert.NoError(t, err, "Shutdown should return no errors") + assert.Empty(t, s.crontab.Entries(), "all jobs should have been removed") +} + +func Test_CanTerminateByCancellingContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + s := NewScheduler(ctx) + s.StartJobEvery(1*time.Minute, func() { fmt.Println("boop") }) + + cancel() + + for i := 0; i < 100; i++ { + if len(s.crontab.Entries()) == 0 { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("all jobs are expected to be cleaned by now; it might be a timing issue, otherwise implementation defect") +} + +func Test_StartAndStopJob(t *testing.T) { + s := NewScheduler(context.Background()) + defer s.Shutdown() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + + var jobOne string + var workDone bool + jobOne = s.StartJobEvery(time.Second, func() { + assert.Equal(t, 1, len(s.crontab.Entries()), "scheduler should have one active job") + workDone = true + + s.StopJob(jobOne) + cancel() + }) + + <-ctx.Done() + assert.True(t, workDone, "value should been set in the job") + assert.Equal(t, 0, len(s.crontab.Entries()), "scheduler should have no active jobs") + +} diff --git a/api/stacks/deploy.go b/api/stacks/deploy.go new file mode 100644 index 000000000..ccf5eb441 --- /dev/null +++ b/api/stacks/deploy.go @@ -0,0 +1,138 @@ +package stacks + +import ( + "strings" + "time" + + "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/http/security" +) + +func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) error { + stack, err := datastore.Stack().Stack(stackID) + if err != nil { + return errors.WithMessagef(err, "failed to get the stack %v", stackID) + } + + if stack.GitConfig == nil { + return nil // do nothing if it isn't a git-based stack + } + + username, password := "", "" + if stack.GitConfig.Authentication != nil { + username, password = stack.GitConfig.Authentication.Username, stack.GitConfig.Authentication.Password + } + + newHash, err := gitService.LatestCommitID(stack.GitConfig.URL, stack.GitConfig.ReferenceName, username, password) + if err != nil { + return errors.WithMessagef(err, "failed to fetch latest commit id of the stack %v", stack.ID) + } + + if strings.EqualFold(newHash, string(stack.GitConfig.ConfigHash)) { + return nil + } + + cloneParams := &cloneRepositoryParameters{ + url: stack.GitConfig.URL, + ref: stack.GitConfig.ReferenceName, + toDir: stack.ProjectPath, + } + if stack.GitConfig.Authentication != nil { + cloneParams.auth = &gitAuth{ + username: username, + password: password, + } + } + + if err := cloneGitRepository(gitService, cloneParams); err != nil { + return errors.WithMessagef(err, "failed to do a fresh clone of the stack %v", stack.ID) + } + + endpoint, err := datastore.Endpoint().Endpoint(stack.EndpointID) + if err != nil { + return errors.WithMessagef(err, "failed to find the endpoint %v associated to the stack %v", stack.EndpointID, stack.ID) + } + + author := stack.UpdatedBy + if author == "" { + author = stack.CreatedBy + } + + registries, err := getUserRegistries(datastore, author, endpoint.ID) + if err != nil { + return err + } + + switch stack.Type { + case portainer.DockerComposeStack: + err := deployer.DeployComposeStack(stack, endpoint, registries) + if err != nil { + return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID) + } + case portainer.DockerSwarmStack: + err := deployer.DeploySwarmStack(stack, endpoint, registries, true) + if err != nil { + return errors.WithMessagef(err, "failed to deploy a docker compose stack %v", stackID) + } + default: + return errors.Errorf("cannot update stack, type %v is unsupported", stack.Type) + } + + stack.UpdateDate = time.Now().Unix() + stack.GitConfig.ConfigHash = newHash + if err := datastore.Stack().UpdateStack(stack.ID, stack); err != nil { + return errors.WithMessagef(err, "failed to update the stack %v", stack.ID) + } + + return nil +} + +func getUserRegistries(datastore portainer.DataStore, authorUsername string, endpointID portainer.EndpointID) ([]portainer.Registry, error) { + registries, err := datastore.Registry().Registries() + if err != nil { + return nil, errors.WithMessage(err, "unable to retrieve registries from the database") + } + + user, err := datastore.User().UserByUsername(authorUsername) + if err != nil { + return nil, errors.WithMessagef(err, "failed to fetch a stack's author [%s]", authorUsername) + } + + if user.Role == portainer.AdministratorRole { + return registries, nil + } + + userMemberships, err := datastore.TeamMembership().TeamMembershipsByUserID(user.ID) + if err != nil { + return nil, errors.WithMessagef(err, "failed to fetch memberships of the stack author [%s]", authorUsername) + } + + filteredRegistries := make([]portainer.Registry, 0, len(registries)) + for _, registry := range registries { + if security.AuthorizedRegistryAccess(®istry, user, userMemberships, endpointID) { + filteredRegistries = append(filteredRegistries, registry) + } + } + + return filteredRegistries, nil +} + +type cloneRepositoryParameters struct { + url string + ref string + toDir string + auth *gitAuth +} + +type gitAuth struct { + username string + password string +} + +func cloneGitRepository(gitService portainer.GitService, cloneParams *cloneRepositoryParameters) error { + if cloneParams.auth != nil { + return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, cloneParams.auth.username, cloneParams.auth.password) + } + return gitService.CloneRepository(cloneParams.toDir, cloneParams.url, cloneParams.ref, "", "") +} diff --git a/api/stacks/deploy_test.go b/api/stacks/deploy_test.go new file mode 100644 index 000000000..58b7de913 --- /dev/null +++ b/api/stacks/deploy_test.go @@ -0,0 +1,221 @@ +package stacks + +import ( + "errors" + "io/ioutil" + "strings" + "testing" + + portainer "github.com/portainer/portainer/api" + bolt "github.com/portainer/portainer/api/bolt/bolttest" + gittypes "github.com/portainer/portainer/api/git/types" + "github.com/stretchr/testify/assert" +) + +type gitService struct { + cloneErr error + id string +} + +func (g *gitService) CloneRepository(destination, repositoryURL, referenceName, username, password string) error { + return g.cloneErr +} + +func (g *gitService) LatestCommitID(repositoryURL, referenceName, username, password string) (string, error) { + return g.id, nil +} + +type noopDeployer struct{} + +func (s *noopDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error { + return nil +} + +func (s *noopDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error { + return nil +} + +func Test_redeployWhenChanged_FailsWhenCannotFindStack(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + err := RedeployWhenChanged(1, nil, store, nil) + assert.Error(t, err) + assert.Truef(t, strings.HasPrefix(err.Error(), "failed to get the stack"), "it isn't an error we expected: %v", err.Error()) +} + +func Test_redeployWhenChanged_DoesNothingWhenNotAGitBasedStack(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + err := store.Stack().CreateStack(&portainer.Stack{ID: 1}) + assert.NoError(t, err, "failed to create a test stack") + + err = RedeployWhenChanged(1, nil, store, &gitService{nil, ""}) + assert.NoError(t, err) +} + +func Test_redeployWhenChanged_DoesNothingWhenNoGitChanges(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + tmpDir, _ := ioutil.TempDir("", "stack") + + err := store.Stack().CreateStack(&portainer.Stack{ + ID: 1, + ProjectPath: tmpDir, + GitConfig: &gittypes.RepoConfig{ + URL: "url", + ReferenceName: "ref", + ConfigHash: "oldHash", + }}) + assert.NoError(t, err, "failed to create a test stack") + + err = RedeployWhenChanged(1, nil, store, &gitService{nil, "oldHash"}) + assert.NoError(t, err) +} + +func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) { + cloneErr := errors.New("failed to clone") + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + err := store.Stack().CreateStack(&portainer.Stack{ + ID: 1, + GitConfig: &gittypes.RepoConfig{ + URL: "url", + ReferenceName: "ref", + ConfigHash: "oldHash", + }}) + assert.NoError(t, err, "failed to create a test stack") + + err = RedeployWhenChanged(1, nil, store, &gitService{cloneErr, "newHash"}) + assert.Error(t, err) + assert.ErrorIs(t, err, cloneErr, "should failed to clone but didn't, check test setup") +} + +func Test_redeployWhenChanged(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + tmpDir, _ := ioutil.TempDir("", "stack") + + err := store.Endpoint().CreateEndpoint(&portainer.Endpoint{ID: 1}) + assert.NoError(t, err, "error creating endpoint") + + username := "user" + err = store.User().CreateUser(&portainer.User{Username: username, Role: portainer.AdministratorRole}) + assert.NoError(t, err, "error creating a user") + + stack := portainer.Stack{ + ID: 1, + EndpointID: 1, + ProjectPath: tmpDir, + UpdatedBy: username, + GitConfig: &gittypes.RepoConfig{ + URL: "url", + ReferenceName: "ref", + ConfigHash: "oldHash", + }} + err = store.Stack().CreateStack(&stack) + assert.NoError(t, err, "failed to create a test stack") + + t.Run("can deploy docker compose stack", func(t *testing.T) { + stack.Type = portainer.DockerComposeStack + store.Stack().UpdateStack(stack.ID, &stack) + + err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) + assert.NoError(t, err) + }) + + t.Run("can deploy docker swarm stack", func(t *testing.T) { + stack.Type = portainer.DockerSwarmStack + store.Stack().UpdateStack(stack.ID, &stack) + + err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) + assert.NoError(t, err) + }) + + t.Run("can NOT deploy kube stack", func(t *testing.T) { + stack.Type = portainer.KubernetesStack + store.Stack().UpdateStack(stack.ID, &stack) + + err = RedeployWhenChanged(1, &noopDeployer{}, store, &gitService{nil, "newHash"}) + assert.EqualError(t, err, "cannot update stack, type 3 is unsupported") + }) +} + +func Test_getUserRegistries(t *testing.T) { + store, teardown := bolt.MustNewTestStore(true) + defer teardown() + + endpointID := 123 + + admin := portainer.User{ID: 1, Username: "admin", Role: portainer.AdministratorRole} + err := store.User().CreateUser(&admin) + assert.NoError(t, err, "error creating an admin") + + user := portainer.User{ID: 2, Username: "user", Role: portainer.StandardUserRole} + err = store.User().CreateUser(&user) + assert.NoError(t, err, "error creating a user") + + team := portainer.Team{ID: 1, Name: "team"} + + store.TeamMembership().CreateTeamMembership(&portainer.TeamMembership{ + ID: 1, + UserID: user.ID, + TeamID: team.ID, + Role: portainer.TeamMember, + }) + + registryReachableByUser := portainer.Registry{ + ID: 1, + RegistryAccesses: portainer.RegistryAccesses{ + portainer.EndpointID(endpointID): { + UserAccessPolicies: map[portainer.UserID]portainer.AccessPolicy{ + user.ID: {RoleID: portainer.RoleID(portainer.StandardUserRole)}, + }, + }, + }, + } + err = store.Registry().CreateRegistry(®istryReachableByUser) + assert.NoError(t, err, "couldn't create a registry") + + registryReachableByTeam := portainer.Registry{ + ID: 2, + RegistryAccesses: portainer.RegistryAccesses{ + portainer.EndpointID(endpointID): { + TeamAccessPolicies: map[portainer.TeamID]portainer.AccessPolicy{ + team.ID: {RoleID: portainer.RoleID(portainer.StandardUserRole)}, + }, + }, + }, + } + err = store.Registry().CreateRegistry(®istryReachableByTeam) + assert.NoError(t, err, "couldn't create a registry") + + registryRestricted := portainer.Registry{ + ID: 3, + RegistryAccesses: portainer.RegistryAccesses{ + portainer.EndpointID(endpointID): { + UserAccessPolicies: map[portainer.UserID]portainer.AccessPolicy{ + user.ID + 100: {RoleID: portainer.RoleID(portainer.StandardUserRole)}, + }, + }, + }, + } + err = store.Registry().CreateRegistry(®istryRestricted) + assert.NoError(t, err, "couldn't create a registry") + + t.Run("admin should has access to all registries", func(t *testing.T) { + registries, err := getUserRegistries(store, admin.Username, portainer.EndpointID(endpointID)) + assert.NoError(t, err) + assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam, registryRestricted}, registries) + }) + + t.Run("regular user has access to registries allowed to him and/or his team", func(t *testing.T) { + registries, err := getUserRegistries(store, user.Username, portainer.EndpointID(endpointID)) + assert.NoError(t, err) + assert.ElementsMatch(t, []portainer.Registry{registryReachableByUser, registryReachableByTeam}, registries) + }) +} diff --git a/api/stacks/deployer.go b/api/stacks/deployer.go new file mode 100644 index 000000000..d38c50cbc --- /dev/null +++ b/api/stacks/deployer.go @@ -0,0 +1,46 @@ +package stacks + +import ( + "sync" + + portainer "github.com/portainer/portainer/api" +) + +type StackDeployer interface { + DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error + DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error +} + +type stackDeployer struct { + lock *sync.Mutex + swarmStackManager portainer.SwarmStackManager + composeStackManager portainer.ComposeStackManager +} + +func NewStackDeployer(swarmStackManager portainer.SwarmStackManager, composeStackManager portainer.ComposeStackManager) *stackDeployer { + return &stackDeployer{ + lock: &sync.Mutex{}, + swarmStackManager: swarmStackManager, + composeStackManager: composeStackManager, + } +} + +func (d *stackDeployer) DeploySwarmStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry, prune bool) error { + d.lock.Lock() + defer d.lock.Unlock() + + d.swarmStackManager.Login(registries, endpoint) + defer d.swarmStackManager.Logout(endpoint) + + return d.swarmStackManager.Deploy(stack, prune, endpoint) +} + +func (d *stackDeployer) DeployComposeStack(stack *portainer.Stack, endpoint *portainer.Endpoint, registries []portainer.Registry) error { + d.lock.Lock() + defer d.lock.Unlock() + + d.swarmStackManager.Login(registries, endpoint) + defer d.swarmStackManager.Logout(endpoint) + + return d.composeStackManager.Up(stack, endpoint) +} diff --git a/api/stacks/scheduled.go b/api/stacks/scheduled.go new file mode 100644 index 000000000..fb90ca22c --- /dev/null +++ b/api/stacks/scheduled.go @@ -0,0 +1,34 @@ +package stacks + +import ( + "log" + "time" + + "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/scheduler" +) + +func StartStackSchedules(scheduler *scheduler.Scheduler, stackdeployer StackDeployer, datastore portainer.DataStore, gitService portainer.GitService) error { + stacks, err := datastore.Stack().RefreshableStacks() + if err != nil { + return errors.Wrap(err, "failed to fetch refreshable stacks") + } + for _, stack := range stacks { + d, err := time.ParseDuration(stack.AutoUpdate.Interval) + if err != nil { + return errors.Wrap(err, "Unable to parse auto update interval") + } + jobID := scheduler.StartJobEvery(d, func() { + if err := RedeployWhenChanged(stack.ID, stackdeployer, datastore, gitService); err != nil { + log.Printf("[ERROR] %s\n", err) + } + }) + + stack.AutoUpdate.JobID = jobID + if err := datastore.Stack().UpdateStack(stack.ID, &stack); err != nil { + return errors.Wrap(err, "failed to update stack job id") + } + } + return nil +}