feat(telemetry): update telemetry job

* feat(api): introduce new datastore interface

* refactor(api): refactor http and main layers

* refactor(api): refactor http and bolt layers

api/authorizations.go

@@ -3,34 +3,13 @@ package portainer
 // AuthorizationService represents a service used to
 // update authorizations associated to a user or team.
 type AuthorizationService struct {
-	endpointService       EndpointService
-	endpointGroupService  EndpointGroupService
-	registryService       RegistryService
-	roleService           RoleService
-	teamMembershipService TeamMembershipService
-	userService           UserService
-}
-
-// AuthorizationServiceParameters are the required parameters
-// used to create a new AuthorizationService.
-type AuthorizationServiceParameters struct {
-	EndpointService       EndpointService
-	EndpointGroupService  EndpointGroupService
-	RegistryService       RegistryService
-	RoleService           RoleService
-	TeamMembershipService TeamMembershipService
-	UserService           UserService
+	dataStore DataStore
 }
 
 // NewAuthorizationService returns a point to a new AuthorizationService instance.
-func NewAuthorizationService(parameters *AuthorizationServiceParameters) *AuthorizationService {
+func NewAuthorizationService(dataStore DataStore) *AuthorizationService {
 	return &AuthorizationService{
-		endpointService:       parameters.EndpointService,
-		endpointGroupService:  parameters.EndpointGroupService,
-		registryService:       parameters.RegistryService,
-		roleService:           parameters.RoleService,
-		teamMembershipService: parameters.TeamMembershipService,
-		userService:           parameters.UserService,
+		dataStore: dataStore,
 	}
 }
 
@@ -449,7 +428,7 @@ func DefaultPortainerAuthorizations() Authorizations {
 // the authorizations will be dropped for the each role. If removeAuthorizations is set to false, the authorizations
 // will be reset based for each role.
 func (service AuthorizationService) UpdateVolumeBrowsingAuthorizations(remove bool) error {
-	roles, err := service.roleService.Roles()
+	roles, err := service.dataStore.Role().Roles()
 	if err != nil {
 		return err
 	}
@@ -459,7 +438,7 @@ func (service AuthorizationService) UpdateVolumeBrowsingAuthorizations(remove bo
 		if role.ID != RoleID(1) {
 			updateRoleVolumeBrowsingAuthorizations(&role, remove)
 
-			err := service.roleService.UpdateRole(role.ID, &role)
+			err := service.dataStore.Role().UpdateRole(role.ID, &role)
 			if err != nil {
 				return err
 			}
@@ -492,7 +471,7 @@ func updateRoleVolumeBrowsingAuthorizations(role *Role, removeAuthorizations boo
 
 // RemoveTeamAccessPolicies will remove all existing access policies associated to the specified team
 func (service *AuthorizationService) RemoveTeamAccessPolicies(teamID TeamID) error {
-	endpoints, err := service.endpointService.Endpoints()
+	endpoints, err := service.dataStore.Endpoint().Endpoints()
 	if err != nil {
 		return err
 	}
@@ -502,7 +481,7 @@ func (service *AuthorizationService) RemoveTeamAccessPolicies(teamID TeamID) err
 			if policyTeamID == teamID {
 				delete(endpoint.TeamAccessPolicies, policyTeamID)
 
-				err := service.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+				err := service.dataStore.Endpoint().UpdateEndpoint(endpoint.ID, &endpoint)
 				if err != nil {
 					return err
 				}
@@ -512,7 +491,7 @@ func (service *AuthorizationService) RemoveTeamAccessPolicies(teamID TeamID) err
 		}
 	}
 
-	endpointGroups, err := service.endpointGroupService.EndpointGroups()
+	endpointGroups, err := service.dataStore.EndpointGroup().EndpointGroups()
 	if err != nil {
 		return err
 	}
@@ -522,7 +501,7 @@ func (service *AuthorizationService) RemoveTeamAccessPolicies(teamID TeamID) err
 			if policyTeamID == teamID {
 				delete(endpointGroup.TeamAccessPolicies, policyTeamID)
 
-				err := service.endpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+				err := service.dataStore.EndpointGroup().UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
 				if err != nil {
 					return err
 				}
@@ -532,7 +511,7 @@ func (service *AuthorizationService) RemoveTeamAccessPolicies(teamID TeamID) err
 		}
 	}
 
-	registries, err := service.registryService.Registries()
+	registries, err := service.dataStore.Registry().Registries()
 	if err != nil {
 		return err
 	}
@@ -542,7 +521,7 @@ func (service *AuthorizationService) RemoveTeamAccessPolicies(teamID TeamID) err
 			if policyTeamID == teamID {
 				delete(registry.TeamAccessPolicies, policyTeamID)
 
-				err := service.registryService.UpdateRegistry(registry.ID, &registry)
+				err := service.dataStore.Registry().UpdateRegistry(registry.ID, &registry)
 				if err != nil {
 					return err
 				}
@@ -557,7 +536,7 @@ func (service *AuthorizationService) RemoveTeamAccessPolicies(teamID TeamID) err
 
 // RemoveUserAccessPolicies will remove all existing access policies associated to the specified user
 func (service *AuthorizationService) RemoveUserAccessPolicies(userID UserID) error {
-	endpoints, err := service.endpointService.Endpoints()
+	endpoints, err := service.dataStore.Endpoint().Endpoints()
 	if err != nil {
 		return err
 	}
@@ -567,7 +546,7 @@ func (service *AuthorizationService) RemoveUserAccessPolicies(userID UserID) err
 			if policyUserID == userID {
 				delete(endpoint.UserAccessPolicies, policyUserID)
 
-				err := service.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+				err := service.dataStore.Endpoint().UpdateEndpoint(endpoint.ID, &endpoint)
 				if err != nil {
 					return err
 				}
@@ -577,7 +556,7 @@ func (service *AuthorizationService) RemoveUserAccessPolicies(userID UserID) err
 		}
 	}
 
-	endpointGroups, err := service.endpointGroupService.EndpointGroups()
+	endpointGroups, err := service.dataStore.EndpointGroup().EndpointGroups()
 	if err != nil {
 		return err
 	}
@@ -587,7 +566,7 @@ func (service *AuthorizationService) RemoveUserAccessPolicies(userID UserID) err
 			if policyUserID == userID {
 				delete(endpointGroup.UserAccessPolicies, policyUserID)
 
-				err := service.endpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+				err := service.dataStore.EndpointGroup().UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
 				if err != nil {
 					return err
 				}
@@ -597,7 +576,7 @@ func (service *AuthorizationService) RemoveUserAccessPolicies(userID UserID) err
 		}
 	}
 
-	registries, err := service.registryService.Registries()
+	registries, err := service.dataStore.Registry().Registries()
 	if err != nil {
 		return err
 	}
@@ -607,7 +586,7 @@ func (service *AuthorizationService) RemoveUserAccessPolicies(userID UserID) err
 			if policyUserID == userID {
 				delete(registry.UserAccessPolicies, policyUserID)
 
-				err := service.registryService.UpdateRegistry(registry.ID, &registry)
+				err := service.dataStore.Registry().UpdateRegistry(registry.ID, &registry)
 				if err != nil {
 					return err
 				}
@@ -622,7 +601,7 @@ func (service *AuthorizationService) RemoveUserAccessPolicies(userID UserID) err
 
 // UpdateUsersAuthorizations will trigger an update of the authorizations for all the users.
 func (service *AuthorizationService) UpdateUsersAuthorizations() error {
-	users, err := service.userService.Users()
+	users, err := service.dataStore.User().Users()
 	if err != nil {
 		return err
 	}
@@ -638,7 +617,7 @@ func (service *AuthorizationService) UpdateUsersAuthorizations() error {
 }
 
 func (service *AuthorizationService) updateUserAuthorizations(userID UserID) error {
-	user, err := service.userService.User(userID)
+	user, err := service.dataStore.User().User(userID)
 	if err != nil {
 		return err
 	}
@@ -650,7 +629,7 @@ func (service *AuthorizationService) updateUserAuthorizations(userID UserID) err
 
 	user.EndpointAuthorizations = endpointAuthorizations
 
-	return service.userService.UpdateUser(userID, user)
+	return service.dataStore.User().UpdateUser(userID, user)
 }
 
 func (service *AuthorizationService) getAuthorizations(user *User) (EndpointAuthorizations, error) {
@@ -659,22 +638,22 @@ func (service *AuthorizationService) getAuthorizations(user *User) (EndpointAuth
 		return endpointAuthorizations, nil
 	}
 
-	userMemberships, err := service.teamMembershipService.TeamMembershipsByUserID(user.ID)
+	userMemberships, err := service.dataStore.TeamMembership().TeamMembershipsByUserID(user.ID)
 	if err != nil {
 		return endpointAuthorizations, err
 	}
 
-	endpoints, err := service.endpointService.Endpoints()
+	endpoints, err := service.dataStore.Endpoint().Endpoints()
 	if err != nil {
 		return endpointAuthorizations, err
 	}
 
-	endpointGroups, err := service.endpointGroupService.EndpointGroups()
+	endpointGroups, err := service.dataStore.EndpointGroup().EndpointGroups()
 	if err != nil {
 		return endpointAuthorizations, err
 	}
 
-	roles, err := service.roleService.Roles()
+	roles, err := service.dataStore.Role().Roles()
 	if err != nil {
 		return endpointAuthorizations, err
 	}

api/bolt/datastore.go

@@ -5,16 +5,14 @@ import (
 	"path"
 	"time"
 
-	"github.com/portainer/portainer/api/bolt/edgegroup"
-	"github.com/portainer/portainer/api/bolt/edgestack"
-	"github.com/portainer/portainer/api/bolt/endpointrelation"
-	"github.com/portainer/portainer/api/bolt/tunnelserver"
-
 	"github.com/boltdb/bolt"
 	"github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/bolt/dockerhub"
+	"github.com/portainer/portainer/api/bolt/edgegroup"
+	"github.com/portainer/portainer/api/bolt/edgestack"
 	"github.com/portainer/portainer/api/bolt/endpoint"
 	"github.com/portainer/portainer/api/bolt/endpointgroup"
+	"github.com/portainer/portainer/api/bolt/endpointrelation"
 	"github.com/portainer/portainer/api/bolt/extension"
 	"github.com/portainer/portainer/api/bolt/migrator"
 	"github.com/portainer/portainer/api/bolt/registry"
@@ -26,7 +24,8 @@ import (
 	"github.com/portainer/portainer/api/bolt/tag"
 	"github.com/portainer/portainer/api/bolt/team"
 	"github.com/portainer/portainer/api/bolt/teammembership"
-	"github.com/portainer/portainer/api/bolt/template"
+	"github.com/portainer/portainer/api/bolt/telemetry"
+	"github.com/portainer/portainer/api/bolt/tunnelserver"
 	"github.com/portainer/portainer/api/bolt/user"
 	"github.com/portainer/portainer/api/bolt/version"
 	"github.com/portainer/portainer/api/bolt/webhook"
@@ -41,9 +40,8 @@ const (
 type Store struct {
 	path                    string
 	db                      *bolt.DB
-	checkForDataMigration   bool
+	isNew                   bool
 	fileService             portainer.FileService
-	RoleService             *role.Service
 	DockerHubService        *dockerhub.Service
 	EdgeGroupService        *edgegroup.Service
 	EdgeStackService        *edgestack.Service
@@ -53,17 +51,18 @@ type Store struct {
 	ExtensionService        *extension.Service
 	RegistryService         *registry.Service
 	ResourceControlService  *resourcecontrol.Service
+	RoleService             *role.Service
+	ScheduleService         *schedule.Service
 	SettingsService         *settings.Service
 	StackService            *stack.Service
 	TagService              *tag.Service
 	TeamMembershipService   *teammembership.Service
 	TeamService             *team.Service
-	TemplateService         *template.Service
+	TelemetryService        *telemetry.Service
 	TunnelServerService     *tunnelserver.Service
 	UserService             *user.Service
 	VersionService          *version.Service
 	WebhookService          *webhook.Service
-	ScheduleService         *schedule.Service
 }
 
 // NewStore initializes a new Store and the associated services
@@ -71,6 +70,7 @@ func NewStore(storePath string, fileService portainer.FileService) (*Store, erro
 	store := &Store{
 		path:        storePath,
 		fileService: fileService,
+		isNew:       true,
 	}
 
 	databasePath := path.Join(storePath, databaseFileName)
@@ -79,10 +79,8 @@ func NewStore(storePath string, fileService portainer.FileService) (*Store, erro
 		return nil, err
 	}
 
-	if !databaseFileExists {
-		store.checkForDataMigration = false
-	} else {
-		store.checkForDataMigration = true
+	if databaseFileExists {
+		store.isNew = false
 	}
 
 	return store, nil
@@ -108,9 +106,16 @@ func (store *Store) Close() error {
 	return nil
 }
 
+// IsNew returns true if the database was just created and false if it is re-using
+// existing data.
+func (store *Store) IsNew() bool {
+	return store.isNew
+}
+
 // MigrateData automatically migrate the data based on the DBVersion.
+// This process is only triggered on an existing database, not if the database was just created.
 func (store *Store) MigrateData() error {
-	if !store.checkForDataMigration {
+	if store.isNew {
 		return store.VersionService.StoreDBVersion(portainer.DBVersion)
 	}
 
@@ -137,10 +142,11 @@ func (store *Store) MigrateData() error {
 			StackService:            store.StackService,
 			TagService:              store.TagService,
 			TeamMembershipService:   store.TeamMembershipService,
-			TemplateService:         store.TemplateService,
+			TelemetryService:        store.TelemetryService,
 			UserService:             store.UserService,
 			VersionService:          store.VersionService,
 			FileService:             store.fileService,
+			AuthorizationService:    portainer.NewAuthorizationService(store),
 		}
 		migrator := migrator.NewMigrator(migratorParams)
 
@@ -246,11 +252,11 @@ func (store *Store) initServices() error {
 	}
 	store.TeamService = teamService
 
-	templateService, err := template.NewService(store.db)
+	telemetryService, err := telemetry.NewService(store.db)
 	if err != nil {
 		return err
 	}
-	store.TemplateService = templateService
+	store.TelemetryService = telemetryService
 
 	tunnelServerService, err := tunnelserver.NewService(store.db)
 	if err != nil {
@@ -284,3 +290,108 @@ func (store *Store) initServices() error {
 
 	return nil
 }
+
+// DockerHub gives access to the DockerHub data management layer
+func (store *Store) DockerHub() portainer.DockerHubService {
+	return store.DockerHubService
+}
+
+// EdgeGroup gives access to the EdgeGroup data management layer
+func (store *Store) EdgeGroup() portainer.EdgeGroupService {
+	return store.EdgeGroupService
+}
+
+// EdgeStack gives access to the EdgeStack data management layer
+func (store *Store) EdgeStack() portainer.EdgeStackService {
+	return store.EdgeStackService
+}
+
+// Endpoint gives access to the Endpoint data management layer
+func (store *Store) Endpoint() portainer.EndpointService {
+	return store.EndpointService
+}
+
+// EndpointGroup gives access to the EndpointGroup data management layer
+func (store *Store) EndpointGroup() portainer.EndpointGroupService {
+	return store.EndpointGroupService
+}
+
+// EndpointRelation gives access to the EndpointRelation data management layer
+func (store *Store) EndpointRelation() portainer.EndpointRelationService {
+	return store.EndpointRelationService
+}
+
+// Extension gives access to the Extension data management layer
+func (store *Store) Extension() portainer.ExtensionService {
+	return store.ExtensionService
+}
+
+// Registry gives access to the Registry data management layer
+func (store *Store) Registry() portainer.RegistryService {
+	return store.RegistryService
+}
+
+// ResourceControl gives access to the ResourceControl data management layer
+func (store *Store) ResourceControl() portainer.ResourceControlService {
+	return store.ResourceControlService
+}
+
+// Role gives access to the Role data management layer
+func (store *Store) Role() portainer.RoleService {
+	return store.RoleService
+}
+
+// Schedule gives access to the Schedule data management layer
+func (store *Store) Schedule() portainer.ScheduleService {
+	return store.ScheduleService
+}
+
+// Settings gives access to the Settings data management layer
+func (store *Store) Settings() portainer.SettingsService {
+	return store.SettingsService
+}
+
+// Stack gives access to the Stack data management layer
+func (store *Store) Stack() portainer.StackService {
+	return store.StackService
+}
+
+// Tag gives access to the Tag data management layer
+func (store *Store) Tag() portainer.TagService {
+	return store.TagService
+}
+
+// TeamMembership gives access to the TeamMembership data management layer
+func (store *Store) TeamMembership() portainer.TeamMembershipService {
+	return store.TeamMembershipService
+}
+
+// Team gives access to the Team data management layer
+func (store *Store) Team() portainer.TeamService {
+	return store.TeamService
+}
+
+// Telemetry gives access to the Telemetry data management layer
+func (store *Store) Telemetry() portainer.TelemetryService {
+	return store.TelemetryService
+}
+
+// TunnelServer gives access to the TunnelServer data management layer
+func (store *Store) TunnelServer() portainer.TunnelServerService {
+	return store.TunnelServerService
+}
+
+// User gives access to the User data management layer
+func (store *Store) User() portainer.UserService {
+	return store.UserService
+}
+
+// Version gives access to the Version data management layer
+func (store *Store) Version() portainer.VersionService {
+	return store.VersionService
+}
+
+// Webhook gives access to the Webhook data management layer
+func (store *Store) Webhook() portainer.WebhookService {
+	return store.WebhookService
+}

api/bolt/init.go

@@ -1,9 +1,62 @@
 package bolt
 
-import portainer "github.com/portainer/portainer/api"
+import (
+	"github.com/gofrs/uuid"
+	portainer "github.com/portainer/portainer/api"
+)
 
 // Init creates the default data set.
 func (store *Store) Init() error {
+	_, err := store.SettingsService.Settings()
+	if err == portainer.ErrObjectNotFound {
+		defaultSettings := &portainer.Settings{
+			AuthenticationMethod: portainer.AuthenticationInternal,
+			BlackListedLabels:    make([]portainer.Pair, 0),
+			LDAPSettings: portainer.LDAPSettings{
+				AnonymousMode:   true,
+				AutoCreateUsers: true,
+				TLSConfig:       portainer.TLSConfiguration{},
+				SearchSettings: []portainer.LDAPSearchSettings{
+					portainer.LDAPSearchSettings{},
+				},
+				GroupSearchSettings: []portainer.LDAPGroupSearchSettings{
+					portainer.LDAPGroupSearchSettings{},
+				},
+			},
+			OAuthSettings:                      portainer.OAuthSettings{},
+			AllowBindMountsForRegularUsers:     true,
+			AllowPrivilegedModeForRegularUsers: true,
+			AllowVolumeBrowserForRegularUsers:  false,
+			EnableHostManagementFeatures:       false,
+			EdgeAgentCheckinInterval:           portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
+			TemplatesURL:                       portainer.DefaultTemplatesURL,
+			Telemetry:                          true,
+		}
+
+		err = store.SettingsService.UpdateSettings(defaultSettings)
+		if err != nil {
+			return err
+		}
+	} else if err != nil {
+		return err
+	}
+
+	_, err = store.DockerHubService.DockerHub()
+	if err == portainer.ErrObjectNotFound {
+		defaultDockerHub := &portainer.DockerHub{
+			Authentication: false,
+			Username:       "",
+			Password:       "",
+		}
+
+		err := store.DockerHubService.UpdateDockerHub(defaultDockerHub)
+		if err != nil {
+			return err
+		}
+	} else if err != nil {
+		return err
+	}
+
 	groups, err := store.EndpointGroupService.EndpointGroups()
 	if err != nil {
 		return err
@@ -80,5 +133,24 @@ func (store *Store) Init() error {
 		}
 	}
 
+	_, err = store.TelemetryService.Telemetry()
+	if err == portainer.ErrObjectNotFound {
+		token, err := uuid.NewV4()
+		if err != nil {
+			return err
+		}
+
+		defaultTelemetry := &portainer.Telemetry{
+			TelemetryID: token.String(),
+		}
+
+		err = store.TelemetryService.Update(defaultTelemetry)
+		if err != nil {
+			return err
+		}
+	} else if err != nil {
+		return err
+	}
+
 	return nil
 }

api/bolt/migrator/migrate_dbversion14.go

@@ -1,11 +1,5 @@
 package migrator
 
-import (
-	"strings"
-
-	"github.com/portainer/portainer/api"
-)
-
 func (m *Migrator) updateSettingsToDBVersion15() error {
 	legacySettings, err := m.settingsService.Settings()
 	if err != nil {
@@ -17,19 +11,6 @@ func (m *Migrator) updateSettingsToDBVersion15() error {
 }
 
 func (m *Migrator) updateTemplatesToVersion15() error {
-	legacyTemplates, err := m.templateService.Templates()
-	if err != nil {
-		return err
-	}
-
-	for _, template := range legacyTemplates {
-		template.Logo = strings.Replace(template.Logo, "https://portainer.io/images", portainer.AssetsServerURL, -1)
-
-		err = m.templateService.UpdateTemplate(template.ID, &template)
-		if err != nil {
-			return err
-		}
-	}
-
+	// Removed with the entire template management layer, part of https://github.com/portainer/portainer/issues/3707
 	return nil
 }

api/bolt/migrator/migrate_dbversion19.go

@@ -7,17 +7,7 @@ import (
 )
 
 func (m *Migrator) updateUsersToDBVersion20() error {
-	authorizationServiceParameters := &portainer.AuthorizationServiceParameters{
-		EndpointService:       m.endpointService,
-		EndpointGroupService:  m.endpointGroupService,
-		RegistryService:       m.registryService,
-		RoleService:           m.roleService,
-		TeamMembershipService: m.teamMembershipService,
-		UserService:           m.userService,
-	}
-
-	authorizationService := portainer.NewAuthorizationService(authorizationServiceParameters)
-	return authorizationService.UpdateUsersAuthorizations()
+	return m.authorizationService.UpdateUsersAuthorizations()
 }
 
 func (m *Migrator) updateSettingsToDBVersion20() error {

api/bolt/migrator/migrate_dbversion20.go

@@ -74,16 +74,9 @@ func (m *Migrator) updateUsersAndRolesToDBVersion22() error {
 	readOnlyUserRole.Authorizations = portainer.DefaultEndpointAuthorizationsForReadOnlyUserRole(settings.AllowVolumeBrowserForRegularUsers)
 
 	err = m.roleService.UpdateRole(readOnlyUserRole.ID, readOnlyUserRole)
-
-	authorizationServiceParameters := &portainer.AuthorizationServiceParameters{
-		EndpointService:       m.endpointService,
-		EndpointGroupService:  m.endpointGroupService,
-		RegistryService:       m.registryService,
-		RoleService:           m.roleService,
-		TeamMembershipService: m.teamMembershipService,
-		UserService:           m.userService,
+	if err != nil {
+		return err
 	}
 
-	authorizationService := portainer.NewAuthorizationService(authorizationServiceParameters)
-	return authorizationService.UpdateUsersAuthorizations()
+	return m.authorizationService.UpdateUsersAuthorizations()
 }

api/bolt/migrator/migrate_dbversion24.go

@@ -0,0 +1,37 @@
+package migrator
+
+import (
+	"github.com/gofrs/uuid"
+	portainer "github.com/portainer/portainer/api"
+)
+
+func (m *Migrator) updateTelemetryToDB24() error {
+	_, err := m.telemetryService.Telemetry()
+	if err == portainer.ErrObjectNotFound {
+		token, err := uuid.NewV4()
+		if err != nil {
+			return err
+		}
+
+		defaultTelemetry := &portainer.Telemetry{
+			TelemetryID: token.String(),
+		}
+
+		return m.telemetryService.Update(defaultTelemetry)
+	} else if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (m *Migrator) updateSettingsToDB24() error {
+	legacySettings, err := m.settingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.Telemetry = true
+
+	return m.settingsService.UpdateSettings(legacySettings)
+}

api/bolt/migrator/migrator.go

@@ -15,7 +15,7 @@ import (
 	"github.com/portainer/portainer/api/bolt/stack"
 	"github.com/portainer/portainer/api/bolt/tag"
 	"github.com/portainer/portainer/api/bolt/teammembership"
-	"github.com/portainer/portainer/api/bolt/template"
+	"github.com/portainer/portainer/api/bolt/telemetry"
 	"github.com/portainer/portainer/api/bolt/user"
 	"github.com/portainer/portainer/api/bolt/version"
 )
@@ -37,10 +37,11 @@ type (
 		stackService            *stack.Service
 		tagService              *tag.Service
 		teamMembershipService   *teammembership.Service
-		templateService         *template.Service
+		telemetryService        *telemetry.Service
 		userService             *user.Service
 		versionService          *version.Service
 		fileService             portainer.FileService
+		authorizationService    *portainer.AuthorizationService
 	}
 
 	// Parameters represents the required parameters to create a new Migrator instance.
@@ -59,10 +60,11 @@ type (
 		StackService            *stack.Service
 		TagService              *tag.Service
 		TeamMembershipService   *teammembership.Service
-		TemplateService         *template.Service
+		TelemetryService        *telemetry.Service
 		UserService             *user.Service
 		VersionService          *version.Service
 		FileService             portainer.FileService
+		AuthorizationService    *portainer.AuthorizationService
 	}
 )
 
@@ -82,17 +84,17 @@ func NewMigrator(parameters *Parameters) *Migrator {
 		settingsService:         parameters.SettingsService,
 		tagService:              parameters.TagService,
 		teamMembershipService:   parameters.TeamMembershipService,
-		templateService:         parameters.TemplateService,
+		telemetryService:        parameters.TelemetryService,
 		stackService:            parameters.StackService,
 		userService:             parameters.UserService,
 		versionService:          parameters.VersionService,
 		fileService:             parameters.FileService,
+		authorizationService:    parameters.AuthorizationService,
 	}
 }
 
 // Migrate checks the database version and migrate the existing data to the most recent data model.
 func (m *Migrator) Migrate() error {
-
 	// Portainer < 1.12
 	if m.currentDBVersion < 1 {
 		err := m.updateAdminUserToDBVersion1()
@@ -309,7 +311,7 @@ func (m *Migrator) Migrate() error {
 		}
 	}
 
-	// Portainer 1.24.0-dev
+	// Portainer 1.24.0
 	if m.currentDBVersion < 23 {
 		err := m.updateTagsToDBVersion23()
 		if err != nil {
@@ -322,5 +324,18 @@ func (m *Migrator) Migrate() error {
 		}
 	}
 
+	// Portainer 2.0
+	if m.currentDBVersion < 24 {
+		err := m.updateTelemetryToDB24()
+		if err != nil {
+			return err
+		}
+
+		err = m.updateSettingsToDB24()
+		if err != nil {
+			return err
+		}
+	}
+
 	return m.versionService.StoreDBVersion(portainer.DBVersion)
 }

api/bolt/telemetry/telemetry.go

@@ -0,0 +1,48 @@
+package telemetry
+
+import (
+	"github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/bolt/internal"
+
+	"github.com/boltdb/bolt"
+)
+
+const (
+	// BucketName represents the name of the bucket where this service stores data.
+	BucketName  = "telemetry"
+	settingsKey = "TELEMETRY"
+)
+
+// Service represents a service for managing endpoint data.
+type Service struct {
+	db *bolt.DB
+}
+
+// NewService creates a new instance of a service.
+func NewService(db *bolt.DB) (*Service, error) {
+	err := internal.CreateBucket(db, BucketName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Service{
+		db: db,
+	}, nil
+}
+
+// Telemetry retrieve the Telemetry object.
+func (service *Service) Telemetry() (*portainer.Telemetry, error) {
+	var telemetry portainer.Telemetry
+
+	err := internal.GetObject(service.db, BucketName, []byte(settingsKey), &telemetry)
+	if err != nil {
+		return nil, err
+	}
+
+	return &telemetry, nil
+}
+
+// Update persists a Telemetry object.
+func (service *Service) Update(telemetry *portainer.Telemetry) error {
+	return internal.UpdateObject(service.db, BucketName, []byte(settingsKey), telemetry)
+}

api/bolt/template/template.go

@@ -1,95 +0,0 @@
-package template
-
-import (
-	"github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/bolt/internal"
-
-	"github.com/boltdb/bolt"
-)
-
-const (
-	// BucketName represents the name of the bucket where this service stores data.
-	BucketName = "templates"
-)
-
-// Service represents a service for managing endpoint data.
-type Service struct {
-	db *bolt.DB
-}
-
-// NewService creates a new instance of a service.
-func NewService(db *bolt.DB) (*Service, error) {
-	err := internal.CreateBucket(db, BucketName)
-	if err != nil {
-		return nil, err
-	}
-
-	return &Service{
-		db: db,
-	}, nil
-}
-
-// Templates return an array containing all the templates.
-func (service *Service) Templates() ([]portainer.Template, error) {
-	var templates = make([]portainer.Template, 0)
-
-	err := service.db.View(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(BucketName))
-
-		cursor := bucket.Cursor()
-		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			var template portainer.Template
-			err := internal.UnmarshalObject(v, &template)
-			if err != nil {
-				return err
-			}
-			templates = append(templates, template)
-		}
-
-		return nil
-	})
-
-	return templates, err
-}
-
-// Template returns a template by ID.
-func (service *Service) Template(ID portainer.TemplateID) (*portainer.Template, error) {
-	var template portainer.Template
-	identifier := internal.Itob(int(ID))
-
-	err := internal.GetObject(service.db, BucketName, identifier, &template)
-	if err != nil {
-		return nil, err
-	}
-
-	return &template, nil
-}
-
-// CreateTemplate creates a new template.
-func (service *Service) CreateTemplate(template *portainer.Template) error {
-	return service.db.Update(func(tx *bolt.Tx) error {
-		bucket := tx.Bucket([]byte(BucketName))
-
-		id, _ := bucket.NextSequence()
-		template.ID = portainer.TemplateID(id)
-
-		data, err := internal.MarshalObject(template)
-		if err != nil {
-			return err
-		}
-
-		return bucket.Put(internal.Itob(int(template.ID)), data)
-	})
-}
-
-// UpdateTemplate saves a template.
-func (service *Service) UpdateTemplate(ID portainer.TemplateID, template *portainer.Template) error {
-	identifier := internal.Itob(int(ID))
-	return internal.UpdateObject(service.db, BucketName, identifier, template)
-}
-
-// DeleteTemplate deletes a template.
-func (service *Service) DeleteTemplate(ID portainer.TemplateID) error {
-	identifier := internal.Itob(int(ID))
-	return internal.DeleteObject(service.db, BucketName, identifier)
-}

api/chisel/service.go

@@ -24,21 +24,19 @@ const (
 // It is used to start a reverse tunnel server and to manage the connection status of each tunnel
 // connected to the tunnel server.
 type Service struct {
-	serverFingerprint   string
-	serverPort          string
-	tunnelDetailsMap    cmap.ConcurrentMap
-	endpointService     portainer.EndpointService
-	tunnelServerService portainer.TunnelServerService
-	snapshotter         portainer.Snapshotter
-	chiselServer        *chserver.Server
+	serverFingerprint string
+	serverPort        string
+	tunnelDetailsMap  cmap.ConcurrentMap
+	dataStore         portainer.DataStore
+	snapshotter       portainer.Snapshotter
+	chiselServer      *chserver.Server
 }
 
 // NewService returns a pointer to a new instance of Service
-func NewService(endpointService portainer.EndpointService, tunnelServerService portainer.TunnelServerService) *Service {
+func NewService(dataStore portainer.DataStore) *Service {
 	return &Service{
-		tunnelDetailsMap:    cmap.New(),
-		endpointService:     endpointService,
-		tunnelServerService: tunnelServerService,
+		tunnelDetailsMap: cmap.New(),
+		dataStore:        dataStore,
 	}
 }
 
@@ -89,7 +87,7 @@ func (service *Service) StartTunnelServer(addr, port string, snapshotter portain
 func (service *Service) retrievePrivateKeySeed() (string, error) {
 	var serverInfo *portainer.TunnelServerInfo
 
-	serverInfo, err := service.tunnelServerService.Info()
+	serverInfo, err := service.dataStore.TunnelServer().Info()
 	if err == portainer.ErrObjectNotFound {
 		keySeed := uniuri.NewLen(16)
 
@@ -97,7 +95,7 @@ func (service *Service) retrievePrivateKeySeed() (string, error) {
 			PrivateKeySeed: keySeed,
 		}
 
-		err := service.tunnelServerService.UpdateInfo(serverInfo)
+		err := service.dataStore.TunnelServer().UpdateInfo(serverInfo)
 		if err != nil {
 			return "", err
 		}
@@ -173,7 +171,7 @@ func (service *Service) checkTunnels() {
 }
 
 func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tunnelPort int) error {
-	endpoint, err := service.endpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
 	if err != nil {
 		return err
 	}
@@ -187,5 +185,5 @@ func (service *Service) snapshotEnvironment(endpointID portainer.EndpointID, tun
 
 	endpoint.Snapshots = []portainer.Snapshot{*snapshot}
 	endpoint.URL = endpointURL
-	return service.endpointService.UpdateEndpoint(endpoint.ID, endpoint)
+	return service.dataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
 }

api/chisel/tunnel.go

@@ -97,7 +97,7 @@ func (service *Service) SetTunnelStatusToRequired(endpointID portainer.EndpointI
 	tunnel := service.GetTunnelDetails(endpointID)
 
 	if tunnel.Port == 0 {
-		endpoint, err := service.endpointService.Endpoint(endpointID)
+		endpoint, err := service.dataStore.Endpoint().Endpoint(endpointID)
 		if err != nil {
 			return err
 		}

api/cli/cli.go

@@ -19,11 +19,7 @@ type Service struct{}
 const (
 	errInvalidEndpointProtocol       = portainer.Error("Invalid endpoint protocol: Portainer only supports unix://, npipe:// or tcp://")
 	errSocketOrNamedPipeNotFound     = portainer.Error("Unable to locate Unix socket or named pipe")
-	errEndpointsFileNotFound         = portainer.Error("Unable to locate external endpoints file")
-	errTemplateFileNotFound          = portainer.Error("Unable to locate template file on disk")
-	errInvalidSyncInterval           = portainer.Error("Invalid synchronization interval")
 	errInvalidSnapshotInterval       = portainer.Error("Invalid snapshot interval")
-	errEndpointExcludeExternal       = portainer.Error("Cannot use the -H flag mutually with --external-endpoints")
 	errNoAuthExcludeAdminPassword    = portainer.Error("Cannot use --no-auth with --admin-password or --admin-password-file")
 	errAdminPassExcludeAdminPassFile = portainer.Error("Cannot use --admin-password with --admin-password-file")
 )
@@ -39,9 +35,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		Assets:            kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
 		Data:              kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
 		EndpointURL:       kingpin.Flag("host", "Endpoint URL").Short('H').String(),
-		ExternalEndpoints: kingpin.Flag("external-endpoints", "Path to a file defining available endpoints (deprecated)").String(),
 		NoAuth:            kingpin.Flag("no-auth", "Disable authentication (deprecated)").Default(defaultNoAuth).Bool(),
-		NoAnalytics:       kingpin.Flag("no-analytics", "Disable Analytics in app").Default(defaultNoAnalytics).Bool(),
 		TLS:               kingpin.Flag("tlsverify", "TLS support").Default(defaultTLS).Bool(),
 		TLSSkipVerify:     kingpin.Flag("tlsskipverify", "Disable TLS server verification").Default(defaultTLSSkipVerify).Bool(),
 		TLSCacert:         kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(),
@@ -50,15 +44,12 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		SSL:               kingpin.Flag("ssl", "Secure Portainer instance using SSL").Default(defaultSSL).Bool(),
 		SSLCert:           kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").Default(defaultSSLCertPath).String(),
 		SSLKey:            kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").Default(defaultSSLKeyPath).String(),
-		SyncInterval:      kingpin.Flag("sync-interval", "Duration between each synchronization via the external endpoints source (deprecated)").Default(defaultSyncInterval).String(),
-		Snapshot:          kingpin.Flag("snapshot", "Start a background job to create endpoint snapshots (deprecated)").Default(defaultSnapshot).Bool(),
 		SnapshotInterval:  kingpin.Flag("snapshot-interval", "Duration between each endpoint snapshot job").Default(defaultSnapshotInterval).String(),
 		AdminPassword:     kingpin.Flag("admin-password", "Hashed admin password").String(),
 		AdminPasswordFile: kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(),
 		Labels:            pairs(kingpin.Flag("hide-label", "Hide containers with a specific label in the UI").Short('l')),
 		Logo:              kingpin.Flag("logo", "URL for the logo displayed in the UI").String(),
 		Templates:         kingpin.Flag("templates", "URL to the templates definitions.").Short('t').String(),
-		TemplateFile:      kingpin.Flag("template-file", "Path to the App templates definitions on the filesystem (deprecated)").Default(defaultTemplateFile).String(),
 	}
 
 	kingpin.Parse()
@@ -79,26 +70,7 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 
 	displayDeprecationWarnings(flags)
 
-	if *flags.EndpointURL != "" && *flags.ExternalEndpoints != "" {
-		return errEndpointExcludeExternal
-	}
-
-	err := validateTemplateFile(*flags.TemplateFile)
-	if err != nil {
-		return err
-	}
-
-	err = validateEndpointURL(*flags.EndpointURL)
-	if err != nil {
-		return err
-	}
-
-	err = validateExternalEndpoints(*flags.ExternalEndpoints)
-	if err != nil {
-		return err
-	}
-
-	err = validateSyncInterval(*flags.SyncInterval)
+	err := validateEndpointURL(*flags.EndpointURL)
 	if err != nil {
 		return err
 	}
@@ -120,25 +92,9 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 }
 
 func displayDeprecationWarnings(flags *portainer.CLIFlags) {
-	if *flags.ExternalEndpoints != "" {
-		log.Println("Warning: the --external-endpoint flag is deprecated and will likely be removed in a future version of Portainer.")
-	}
-
-	if *flags.SyncInterval != defaultSyncInterval {
-		log.Println("Warning: the --sync-interval flag is deprecated and will likely be removed in a future version of Portainer.")
-	}
-
 	if *flags.NoAuth {
 		log.Println("Warning: the --no-auth flag is deprecated and will likely be removed in a future version of Portainer.")
 	}
-
-	if !*flags.Snapshot {
-		log.Println("Warning: the --no-snapshot flag is deprecated and will likely be removed in a future version of Portainer.")
-	}
-
-	if *flags.TemplateFile != "" {
-		log.Println("Warning: the --template-file flag is deprecated and will likely be removed in a future version of Portainer.")
-	}
 }
 
 func validateEndpointURL(endpointURL string) error {
@@ -161,38 +117,6 @@ func validateEndpointURL(endpointURL string) error {
 	return nil
 }
 
-func validateExternalEndpoints(externalEndpoints string) error {
-	if externalEndpoints != "" {
-		if _, err := os.Stat(externalEndpoints); err != nil {
-			if os.IsNotExist(err) {
-				return errEndpointsFileNotFound
-			}
-			return err
-		}
-	}
-	return nil
-}
-
-func validateTemplateFile(templateFile string) error {
-	if _, err := os.Stat(templateFile); err != nil {
-		if os.IsNotExist(err) {
-			return errTemplateFileNotFound
-		}
-		return err
-	}
-	return nil
-}
-
-func validateSyncInterval(syncInterval string) error {
-	if syncInterval != defaultSyncInterval {
-		_, err := time.ParseDuration(syncInterval)
-		if err != nil {
-			return errInvalidSyncInterval
-		}
-	}
-	return nil
-}
-
 func validateSnapshotInterval(snapshotInterval string) error {
 	if snapshotInterval != defaultSnapshotInterval {
 		_, err := time.ParseDuration(snapshotInterval)

api/cli/defaults.go

@@ -9,7 +9,6 @@ const (
 	defaultDataDirectory       = "/data"
 	defaultAssetsDirectory     = "./"
 	defaultNoAuth              = "false"
-	defaultNoAnalytics         = "false"
 	defaultTLS                 = "false"
 	defaultTLSSkipVerify       = "false"
 	defaultTLSCACertPath       = "/certs/ca.pem"
@@ -18,8 +17,5 @@ const (
 	defaultSSL                 = "false"
 	defaultSSLCertPath         = "/certs/portainer.crt"
 	defaultSSLKeyPath          = "/certs/portainer.key"
-	defaultSyncInterval        = "60s"
-	defaultSnapshot            = "true"
 	defaultSnapshotInterval    = "5m"
-	defaultTemplateFile        = "/templates.json"
 )

api/cli/defaults_windows.go

@@ -7,7 +7,6 @@ const (
 	defaultDataDirectory       = "C:\\data"
 	defaultAssetsDirectory     = "./"
 	defaultNoAuth              = "false"
-	defaultNoAnalytics         = "false"
 	defaultTLS                 = "false"
 	defaultTLSSkipVerify       = "false"
 	defaultTLSCACertPath       = "C:\\certs\\ca.pem"
@@ -16,8 +15,5 @@ const (
 	defaultSSL                 = "false"
 	defaultSSLCertPath         = "C:\\certs\\portainer.crt"
 	defaultSSLKeyPath          = "C:\\certs\\portainer.key"
-	defaultSyncInterval        = "60s"
-	defaultSnapshot            = "true"
 	defaultSnapshotInterval    = "5m"
-	defaultTemplateFile        = "/templates.json"
 )

api/cmd/portainer/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"encoding/json"
 	"log"
 	"os"
 	"strings"
@@ -26,13 +25,13 @@ import (
 )
 
 func initCLI() *portainer.CLIFlags {
-	var cli portainer.CLIService = &cli.Service{}
-	flags, err := cli.ParseFlags(portainer.APIVersion)
+	var cliService portainer.CLIService = &cli.Service{}
+	flags, err := cliService.ParseFlags(portainer.APIVersion)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	err = cli.ValidateFlags(flags)
+	err = cliService.ValidateFlags(flags)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -47,7 +46,7 @@ func initFileService(dataStorePath string) portainer.FileService {
 	return fileService
 }
 
-func initStore(dataStorePath string, fileService portainer.FileService) *bolt.Store {
+func initDataStore(dataStorePath string, fileService portainer.FileService) portainer.DataStore {
 	store, err := bolt.NewStore(dataStorePath, fileService)
 	if err != nil {
 		log.Fatal(err)
@@ -117,13 +116,59 @@ func initJobScheduler() portainer.JobScheduler {
 	return cron.NewJobScheduler()
 }
 
-func loadSnapshotSystemSchedule(jobScheduler portainer.JobScheduler, snapshotter portainer.Snapshotter, scheduleService portainer.ScheduleService, endpointService portainer.EndpointService, settingsService portainer.SettingsService) error {
-	settings, err := settingsService.Settings()
+func loadTelemetrySystemSchedule(jobScheduler portainer.JobScheduler, dataStore portainer.DataStore) error {
+	settings, err := dataStore.Settings().Settings()
 	if err != nil {
 		return err
 	}
 
-	schedules, err := scheduleService.SchedulesByJobType(portainer.SnapshotJobType)
+	if !settings.Telemetry {
+		return nil
+	}
+
+	schedules, err := dataStore.Schedule().SchedulesByJobType(portainer.TelemetryJobType)
+	if err != nil {
+		return err
+	}
+
+	var telemetrySchedule *portainer.Schedule
+	if len(schedules) == 0 {
+		telemetryJob := &portainer.TelemetryJob{}
+		telemetrySchedule = &portainer.Schedule{
+			ID:             portainer.ScheduleID(dataStore.Schedule().GetNextIdentifier()),
+			Name:           "system_telemetry",
+			CronExpression: "@daily",
+			Recurring:      true,
+			JobType:        portainer.TelemetryJobType,
+			TelemetryJob:   telemetryJob,
+			Created:        time.Now().Unix(),
+		}
+	} else {
+		telemetrySchedule = &schedules[0]
+	}
+
+	telemetryJobContext := cron.NewTelemetryJobContext(dataStore)
+	telemetryJobRunner := cron.NewTelemetryJobRunner(telemetrySchedule, telemetryJobContext)
+
+	err = jobScheduler.ScheduleJob(telemetryJobRunner)
+	if err != nil {
+		return err
+	}
+
+	if len(schedules) == 0 {
+		return dataStore.Schedule().CreateSchedule(telemetrySchedule)
+	}
+
+	return nil
+}
+
+func loadSnapshotSystemSchedule(jobScheduler portainer.JobScheduler, snapshotter portainer.Snapshotter, dataStore portainer.DataStore) error {
+	settings, err := dataStore.Settings().Settings()
+	if err != nil {
+		return err
+	}
+
+	schedules, err := dataStore.Schedule().SchedulesByJobType(portainer.SnapshotJobType)
 	if err != nil {
 		return err
 	}
@@ -132,7 +177,7 @@ func loadSnapshotSystemSchedule(jobScheduler portainer.JobScheduler, snapshotter
 	if len(schedules) == 0 {
 		snapshotJob := &portainer.SnapshotJob{}
 		snapshotSchedule = &portainer.Schedule{
-			ID:             portainer.ScheduleID(scheduleService.GetNextIdentifier()),
+			ID:             portainer.ScheduleID(dataStore.Schedule().GetNextIdentifier()),
 			Name:           "system_snapshot",
 			CronExpression: "@every " + settings.SnapshotInterval,
 			Recurring:      true,
@@ -144,7 +189,7 @@ func loadSnapshotSystemSchedule(jobScheduler portainer.JobScheduler, snapshotter
 		snapshotSchedule = &schedules[0]
 	}
 
-	snapshotJobContext := cron.NewSnapshotJobContext(endpointService, snapshotter)
+	snapshotJobContext := cron.NewSnapshotJobContext(dataStore, snapshotter)
 	snapshotJobRunner := cron.NewSnapshotJobRunner(snapshotSchedule, snapshotJobContext)
 
 	err = jobScheduler.ScheduleJob(snapshotJobRunner)
@@ -153,52 +198,13 @@ func loadSnapshotSystemSchedule(jobScheduler portainer.JobScheduler, snapshotter
 	}
 
 	if len(schedules) == 0 {
-		return scheduleService.CreateSchedule(snapshotSchedule)
+		return dataStore.Schedule().CreateSchedule(snapshotSchedule)
 	}
 	return nil
 }
 
-func loadEndpointSyncSystemSchedule(jobScheduler portainer.JobScheduler, scheduleService portainer.ScheduleService, endpointService portainer.EndpointService, flags *portainer.CLIFlags) error {
-	if *flags.ExternalEndpoints == "" {
-		return nil
-	}
-
-	log.Println("Using external endpoint definition. Endpoint management via the API will be disabled.")
-
-	schedules, err := scheduleService.SchedulesByJobType(portainer.EndpointSyncJobType)
-	if err != nil {
-		return err
-	}
-
-	if len(schedules) != 0 {
-		return nil
-	}
-
-	endpointSyncJob := &portainer.EndpointSyncJob{}
-
-	endpointSyncSchedule := &portainer.Schedule{
-		ID:              portainer.ScheduleID(scheduleService.GetNextIdentifier()),
-		Name:            "system_endpointsync",
-		CronExpression:  "@every " + *flags.SyncInterval,
-		Recurring:       true,
-		JobType:         portainer.EndpointSyncJobType,
-		EndpointSyncJob: endpointSyncJob,
-		Created:         time.Now().Unix(),
-	}
-
-	endpointSyncJobContext := cron.NewEndpointSyncJobContext(endpointService, *flags.ExternalEndpoints)
-	endpointSyncJobRunner := cron.NewEndpointSyncJobRunner(endpointSyncSchedule, endpointSyncJobContext)
-
-	err = jobScheduler.ScheduleJob(endpointSyncJobRunner)
-	if err != nil {
-		return err
-	}
-
-	return scheduleService.CreateSchedule(endpointSyncSchedule)
-}
-
-func loadSchedulesFromDatabase(jobScheduler portainer.JobScheduler, jobService portainer.JobService, scheduleService portainer.ScheduleService, endpointService portainer.EndpointService, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) error {
-	schedules, err := scheduleService.Schedules()
+func loadSchedulesFromDatabase(jobScheduler portainer.JobScheduler, jobService portainer.JobService, dataStore portainer.DataStore, fileService portainer.FileService, reverseTunnelService portainer.ReverseTunnelService) error {
+	schedules, err := dataStore.Schedule().Schedules()
 	if err != nil {
 		return err
 	}
@@ -206,7 +212,7 @@ func loadSchedulesFromDatabase(jobScheduler portainer.JobScheduler, jobService p
 	for _, schedule := range schedules {
 
 		if schedule.JobType == portainer.ScriptExecutionJobType {
-			jobContext := cron.NewScriptExecutionJobContext(jobService, endpointService, fileService)
+			jobContext := cron.NewScriptExecutionJobContext(jobService, dataStore, fileService)
 			jobRunner := cron.NewScriptExecutionJobRunner(&schedule, jobContext)
 
 			err = jobScheduler.ScheduleJob(jobRunner)
@@ -226,121 +232,31 @@ func loadSchedulesFromDatabase(jobScheduler portainer.JobScheduler, jobService p
 	return nil
 }
 
-func initStatus(endpointManagement, snapshot bool, flags *portainer.CLIFlags) *portainer.Status {
+func initStatus(flags *portainer.CLIFlags) *portainer.Status {
 	return &portainer.Status{
-		Analytics:          !*flags.NoAnalytics,
-		Authentication:     !*flags.NoAuth,
-		EndpointManagement: endpointManagement,
-		Snapshot:           snapshot,
-		Version:            portainer.APIVersion,
+		Authentication: !*flags.NoAuth,
+		Version:        portainer.APIVersion,
 	}
 }
 
-func initDockerHub(dockerHubService portainer.DockerHubService) error {
-	_, err := dockerHubService.DockerHub()
-	if err == portainer.ErrObjectNotFound {
-		dockerhub := &portainer.DockerHub{
-			Authentication: false,
-			Username:       "",
-			Password:       "",
-		}
-		return dockerHubService.UpdateDockerHub(dockerhub)
-	} else if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func initSettings(settingsService portainer.SettingsService, flags *portainer.CLIFlags) error {
-	_, err := settingsService.Settings()
-	if err == portainer.ErrObjectNotFound {
-		settings := &portainer.Settings{
-			LogoURL:              *flags.Logo,
-			AuthenticationMethod: portainer.AuthenticationInternal,
-			LDAPSettings: portainer.LDAPSettings{
-				AnonymousMode:   true,
-				AutoCreateUsers: true,
-				TLSConfig:       portainer.TLSConfiguration{},
-				SearchSettings: []portainer.LDAPSearchSettings{
-					portainer.LDAPSearchSettings{},
-				},
-				GroupSearchSettings: []portainer.LDAPGroupSearchSettings{
-					portainer.LDAPGroupSearchSettings{},
-				},
-			},
-			OAuthSettings:                      portainer.OAuthSettings{},
-			AllowBindMountsForRegularUsers:     true,
-			AllowPrivilegedModeForRegularUsers: true,
-			AllowVolumeBrowserForRegularUsers:  false,
-			EnableHostManagementFeatures:       false,
-			SnapshotInterval:                   *flags.SnapshotInterval,
-			EdgeAgentCheckinInterval:           portainer.DefaultEdgeAgentCheckinIntervalInSeconds,
-		}
-
-		if *flags.Templates != "" {
-			settings.TemplatesURL = *flags.Templates
-		}
-
-		if *flags.Labels != nil {
-			settings.BlackListedLabels = *flags.Labels
-		} else {
-			settings.BlackListedLabels = make([]portainer.Pair, 0)
-		}
-
-		return settingsService.UpdateSettings(settings)
-	} else if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func initTemplates(templateService portainer.TemplateService, fileService portainer.FileService, templateURL, templateFile string) error {
-	if templateURL != "" {
-		log.Printf("Portainer started with the --templates flag. Using external templates, template management will be disabled.")
-		return nil
-	}
-
-	existingTemplates, err := templateService.Templates()
+func updateSettingsFromFlags(dataStore portainer.DataStore, flags *portainer.CLIFlags) error {
+	settings, err := dataStore.Settings().Settings()
 	if err != nil {
 		return err
 	}
 
-	if len(existingTemplates) != 0 {
-		log.Printf("Templates already registered inside the database. Skipping template import.")
-		return nil
+	settings.LogoURL = *flags.Logo
+	settings.SnapshotInterval = *flags.SnapshotInterval
+
+	if *flags.Templates != "" {
+		settings.TemplatesURL = *flags.Templates
 	}
 
-	templatesJSON, err := fileService.GetFileContent(templateFile)
-	if err != nil {
-		log.Println("Unable to retrieve template definitions via filesystem")
-		return err
+	if *flags.Labels != nil {
+		settings.BlackListedLabels = *flags.Labels
 	}
 
-	var templates []portainer.Template
-	err = json.Unmarshal(templatesJSON, &templates)
-	if err != nil {
-		log.Println("Unable to parse templates file. Please review your template definition file.")
-		return err
-	}
-
-	for _, template := range templates {
-		err := templateService.CreateTemplate(&template)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func retrieveFirstEndpointFromDatabase(endpointService portainer.EndpointService) *portainer.Endpoint {
-	endpoints, err := endpointService.Endpoints()
-	if err != nil {
-		log.Fatal(err)
-	}
-	return &endpoints[0]
+	return dataStore.Settings().UpdateSettings(settings)
 }
 
 func loadAndParseKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
@@ -372,7 +288,7 @@ func initKeyPair(fileService portainer.FileService, signatureService portainer.D
 	return generateAndStoreKeyPair(fileService, signatureService)
 }
 
-func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
+func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore portainer.DataStore, snapshotter portainer.Snapshotter) error {
 	tlsConfiguration := portainer.TLSConfiguration{
 		TLS:           *flags.TLS,
 		TLSSkipVerify: *flags.TLSSkipVerify,
@@ -386,7 +302,7 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portain
 		tlsConfiguration.TLS = true
 	}
 
-	endpointID := endpointService.GetNextIdentifier()
+	endpointID := dataStore.Endpoint().GetNextIdentifier()
 	endpoint := &portainer.Endpoint{
 		ID:                 portainer.EndpointID(endpointID),
 		Name:               "primary",
@@ -418,10 +334,10 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portain
 		}
 	}
 
-	return snapshotAndPersistEndpoint(endpoint, endpointService, snapshotter)
+	return snapshotAndPersistEndpoint(endpoint, dataStore, snapshotter)
 }
 
-func createUnsecuredEndpoint(endpointURL string, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
+func createUnsecuredEndpoint(endpointURL string, dataStore portainer.DataStore, snapshotter portainer.Snapshotter) error {
 	if strings.HasPrefix(endpointURL, "tcp://") {
 		_, err := client.ExecutePingOperation(endpointURL, nil)
 		if err != nil {
@@ -429,7 +345,7 @@ func createUnsecuredEndpoint(endpointURL string, endpointService portainer.Endpo
 		}
 	}
 
-	endpointID := endpointService.GetNextIdentifier()
+	endpointID := dataStore.Endpoint().GetNextIdentifier()
 	endpoint := &portainer.Endpoint{
 		ID:                 portainer.EndpointID(endpointID),
 		Name:               "primary",
@@ -445,10 +361,10 @@ func createUnsecuredEndpoint(endpointURL string, endpointService portainer.Endpo
 		Snapshots:          []portainer.Snapshot{},
 	}
 
-	return snapshotAndPersistEndpoint(endpoint, endpointService, snapshotter)
+	return snapshotAndPersistEndpoint(endpoint, dataStore, snapshotter)
 }
 
-func snapshotAndPersistEndpoint(endpoint *portainer.Endpoint, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
+func snapshotAndPersistEndpoint(endpoint *portainer.Endpoint, dataStore portainer.DataStore, snapshotter portainer.Snapshotter) error {
 	snapshot, err := snapshotter.CreateSnapshot(endpoint)
 	endpoint.Status = portainer.EndpointStatusUp
 	if err != nil {
@@ -459,15 +375,15 @@ func snapshotAndPersistEndpoint(endpoint *portainer.Endpoint, endpointService po
 		endpoint.Snapshots = []portainer.Snapshot{*snapshot}
 	}
 
-	return endpointService.CreateEndpoint(endpoint)
+	return dataStore.Endpoint().CreateEndpoint(endpoint)
 }
 
-func initEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) error {
+func initEndpoint(flags *portainer.CLIFlags, dataStore portainer.DataStore, snapshotter portainer.Snapshotter) error {
 	if *flags.EndpointURL == "" {
 		return nil
 	}
 
-	endpoints, err := endpointService.Endpoints()
+	endpoints, err := dataStore.Endpoint().Endpoints()
 	if err != nil {
 		return err
 	}
@@ -478,17 +394,17 @@ func initEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointS
 	}
 
 	if *flags.TLS || *flags.TLSSkipVerify {
-		return createTLSSecuredEndpoint(flags, endpointService, snapshotter)
+		return createTLSSecuredEndpoint(flags, dataStore, snapshotter)
 	}
-	return createUnsecuredEndpoint(*flags.EndpointURL, endpointService, snapshotter)
+	return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotter)
 }
 
 func initJobService(dockerClientFactory *docker.ClientFactory) portainer.JobService {
 	return docker.NewJobService(dockerClientFactory)
 }
 
-func initExtensionManager(fileService portainer.FileService, extensionService portainer.ExtensionService) (portainer.ExtensionManager, error) {
-	extensionManager := exec.NewExtensionManager(fileService, extensionService)
+func initExtensionManager(fileService portainer.FileService, dataStore portainer.DataStore) (portainer.ExtensionManager, error) {
+	extensionManager := exec.NewExtensionManager(fileService, dataStore)
 
 	err := extensionManager.StartExtensions()
 	if err != nil {
@@ -498,11 +414,11 @@ func initExtensionManager(fileService portainer.FileService, extensionService po
 	return extensionManager, nil
 }
 
-func terminateIfNoAdminCreated(userService portainer.UserService) {
+func terminateIfNoAdminCreated(dataStore portainer.DataStore) {
 	timer1 := time.NewTimer(5 * time.Minute)
 	<-timer1.C
 
-	users, err := userService.UsersByRole(portainer.AdministratorRole)
+	users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -518,8 +434,8 @@ func main() {
 
 	fileService := initFileService(*flags.Data)
 
-	store := initStore(*flags.Data, fileService)
-	defer store.Close()
+	dataStore := initDataStore(*flags.Data, fileService)
+	defer dataStore.Close()
 
 	jwtService := initJWTService(!*flags.NoAuth)
 
@@ -536,12 +452,12 @@ func main() {
 		log.Fatal(err)
 	}
 
-	extensionManager, err := initExtensionManager(fileService, store.ExtensionService)
+	extensionManager, err := initExtensionManager(fileService, dataStore)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	reverseTunnelService := chisel.NewService(store.EndpointService, store.TunnelServerService)
+	reverseTunnelService := chisel.NewService(dataStore)
 
 	clientFactory := initClientFactory(digitalSignatureService, reverseTunnelService)
 
@@ -549,11 +465,6 @@ func main() {
 
 	snapshotter := initSnapshotter(clientFactory)
 
-	endpointManagement := true
-	if *flags.ExternalEndpoints != "" {
-		endpointManagement = false
-	}
-
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, *flags.Data, digitalSignatureService, fileService, reverseTunnelService)
 	if err != nil {
 		log.Fatal(err)
@@ -561,45 +472,35 @@ func main() {
 
 	composeStackManager := initComposeStackManager(*flags.Data, reverseTunnelService)
 
-	err = initTemplates(store.TemplateService, fileService, *flags.Templates, *flags.TemplateFile)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = initSettings(store.SettingsService, flags)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	jobScheduler := initJobScheduler()
-
-	err = loadSchedulesFromDatabase(jobScheduler, jobService, store.ScheduleService, store.EndpointService, fileService, reverseTunnelService)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	err = loadEndpointSyncSystemSchedule(jobScheduler, store.ScheduleService, store.EndpointService, flags)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	if *flags.Snapshot {
-		err = loadSnapshotSystemSchedule(jobScheduler, snapshotter, store.ScheduleService, store.EndpointService, store.SettingsService)
+	if dataStore.IsNew() {
+		err = updateSettingsFromFlags(dataStore, flags)
 		if err != nil {
 			log.Fatal(err)
 		}
 	}
 
-	jobScheduler.Start()
+	jobScheduler := initJobScheduler()
 
-	err = initDockerHub(store.DockerHubService)
+	err = loadSchedulesFromDatabase(jobScheduler, jobService, dataStore, fileService, reverseTunnelService)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	applicationStatus := initStatus(endpointManagement, *flags.Snapshot, flags)
+	err = loadSnapshotSystemSchedule(jobScheduler, snapshotter, dataStore)
+	if err != nil {
+		log.Fatal(err)
+	}
 
-	err = initEndpoint(flags, store.EndpointService, snapshotter)
+	err = loadTelemetrySystemSchedule(jobScheduler, dataStore)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	jobScheduler.Start()
+
+	applicationStatus := initStatus(flags)
+
+	err = initEndpoint(flags, dataStore, snapshotter)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -619,7 +520,7 @@ func main() {
 	}
 
 	if adminPasswordHash != "" {
-		users, err := store.UserService.UsersByRole(portainer.AdministratorRole)
+		users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -632,7 +533,7 @@ func main() {
 				Password:                adminPasswordHash,
 				PortainerAuthorizations: portainer.DefaultPortainerAuthorizations(),
 			}
-			err := store.UserService.CreateUser(user)
+			err := dataStore.User().CreateUser(user)
 			if err != nil {
 				log.Fatal(err)
 			}
@@ -642,7 +543,7 @@ func main() {
 	}
 
 	if !*flags.NoAuth {
-		go terminateIfNoAdminCreated(store.UserService)
+		go terminateIfNoAdminCreated(dataStore)
 	}
 
 	err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotter)
@@ -651,47 +552,28 @@ func main() {
 	}
 
 	var server portainer.Server = &http.Server{
-		ReverseTunnelService:    reverseTunnelService,
-		Status:                  applicationStatus,
-		BindAddress:             *flags.Addr,
-		AssetsPath:              *flags.Assets,
-		AuthDisabled:            *flags.NoAuth,
-		EndpointManagement:      endpointManagement,
-		RoleService:             store.RoleService,
-		UserService:             store.UserService,
-		TeamService:             store.TeamService,
-		TeamMembershipService:   store.TeamMembershipService,
-		EdgeGroupService:        store.EdgeGroupService,
-		EdgeStackService:        store.EdgeStackService,
-		EndpointService:         store.EndpointService,
-		EndpointGroupService:    store.EndpointGroupService,
-		EndpointRelationService: store.EndpointRelationService,
-		ExtensionService:        store.ExtensionService,
-		ResourceControlService:  store.ResourceControlService,
-		SettingsService:         store.SettingsService,
-		RegistryService:         store.RegistryService,
-		DockerHubService:        store.DockerHubService,
-		StackService:            store.StackService,
-		ScheduleService:         store.ScheduleService,
-		TagService:              store.TagService,
-		TemplateService:         store.TemplateService,
-		WebhookService:          store.WebhookService,
-		SwarmStackManager:       swarmStackManager,
-		ComposeStackManager:     composeStackManager,
-		ExtensionManager:        extensionManager,
-		CryptoService:           cryptoService,
-		JWTService:              jwtService,
-		FileService:             fileService,
-		LDAPService:             ldapService,
-		GitService:              gitService,
-		SignatureService:        digitalSignatureService,
-		JobScheduler:            jobScheduler,
-		Snapshotter:             snapshotter,
-		SSL:                     *flags.SSL,
-		SSLCert:                 *flags.SSLCert,
-		SSLKey:                  *flags.SSLKey,
-		DockerClientFactory:     clientFactory,
-		JobService:              jobService,
+		ReverseTunnelService: reverseTunnelService,
+		Status:               applicationStatus,
+		BindAddress:          *flags.Addr,
+		AssetsPath:           *flags.Assets,
+		AuthDisabled:         *flags.NoAuth,
+		DataStore:            dataStore,
+		SwarmStackManager:    swarmStackManager,
+		ComposeStackManager:  composeStackManager,
+		ExtensionManager:     extensionManager,
+		CryptoService:        cryptoService,
+		JWTService:           jwtService,
+		FileService:          fileService,
+		LDAPService:          ldapService,
+		GitService:           gitService,
+		SignatureService:     digitalSignatureService,
+		JobScheduler:         jobScheduler,
+		Snapshotter:          snapshotter,
+		SSL:                  *flags.SSL,
+		SSLCert:              *flags.SSLCert,
+		SSLKey:               *flags.SSLKey,
+		DockerClientFactory:  clientFactory,
+		JobService:           jobService,
 	}
 
 	log.Printf("Starting Portainer %s on %s", portainer.APIVersion, *flags.Addr)

api/cron/job_endpoint_sync.go

@@ -1,214 +0,0 @@
-package cron
-
-import (
-	"encoding/json"
-	"io/ioutil"
-	"log"
-	"strings"
-
-	"github.com/portainer/portainer/api"
-)
-
-// EndpointSyncJobRunner is used to run a EndpointSyncJob
-type EndpointSyncJobRunner struct {
-	schedule *portainer.Schedule
-	context  *EndpointSyncJobContext
-}
-
-// EndpointSyncJobContext represents the context of execution of a EndpointSyncJob
-type EndpointSyncJobContext struct {
-	endpointService  portainer.EndpointService
-	endpointFilePath string
-}
-
-// NewEndpointSyncJobContext returns a new context that can be used to execute a EndpointSyncJob
-func NewEndpointSyncJobContext(endpointService portainer.EndpointService, endpointFilePath string) *EndpointSyncJobContext {
-	return &EndpointSyncJobContext{
-		endpointService:  endpointService,
-		endpointFilePath: endpointFilePath,
-	}
-}
-
-// NewEndpointSyncJobRunner returns a new runner that can be scheduled
-func NewEndpointSyncJobRunner(schedule *portainer.Schedule, context *EndpointSyncJobContext) *EndpointSyncJobRunner {
-	return &EndpointSyncJobRunner{
-		schedule: schedule,
-		context:  context,
-	}
-}
-
-type synchronization struct {
-	endpointsToCreate []*portainer.Endpoint
-	endpointsToUpdate []*portainer.Endpoint
-	endpointsToDelete []*portainer.Endpoint
-}
-
-type fileEndpoint struct {
-	Name          string `json:"Name"`
-	URL           string `json:"URL"`
-	TLS           bool   `json:"TLS,omitempty"`
-	TLSSkipVerify bool   `json:"TLSSkipVerify,omitempty"`
-	TLSCACert     string `json:"TLSCACert,omitempty"`
-	TLSCert       string `json:"TLSCert,omitempty"`
-	TLSKey        string `json:"TLSKey,omitempty"`
-}
-
-// GetSchedule returns the schedule associated to the runner
-func (runner *EndpointSyncJobRunner) GetSchedule() *portainer.Schedule {
-	return runner.schedule
-}
-
-// Run triggers the execution of the endpoint synchronization process.
-func (runner *EndpointSyncJobRunner) Run() {
-	data, err := ioutil.ReadFile(runner.context.endpointFilePath)
-	if endpointSyncError(err) {
-		return
-	}
-
-	var fileEndpoints []fileEndpoint
-	err = json.Unmarshal(data, &fileEndpoints)
-	if endpointSyncError(err) {
-		return
-	}
-
-	if len(fileEndpoints) == 0 {
-		log.Println("background job error (endpoint synchronization). External endpoint source is empty")
-		return
-	}
-
-	storedEndpoints, err := runner.context.endpointService.Endpoints()
-	if endpointSyncError(err) {
-		return
-	}
-
-	convertedFileEndpoints := convertFileEndpoints(fileEndpoints)
-
-	sync := prepareSyncData(storedEndpoints, convertedFileEndpoints)
-	if sync.requireSync() {
-		err = runner.context.endpointService.Synchronize(sync.endpointsToCreate, sync.endpointsToUpdate, sync.endpointsToDelete)
-		if endpointSyncError(err) {
-			return
-		}
-		log.Printf("Endpoint synchronization ended. [created: %v] [updated: %v] [deleted: %v]", len(sync.endpointsToCreate), len(sync.endpointsToUpdate), len(sync.endpointsToDelete))
-	}
-}
-
-func endpointSyncError(err error) bool {
-	if err != nil {
-		log.Printf("background job error (endpoint synchronization). Unable to synchronize endpoints (err=%s)\n", err)
-		return true
-	}
-	return false
-}
-
-func isValidEndpoint(endpoint *portainer.Endpoint) bool {
-	if endpoint.Name != "" && endpoint.URL != "" {
-		if !strings.HasPrefix(endpoint.URL, "unix://") && !strings.HasPrefix(endpoint.URL, "tcp://") {
-			return false
-		}
-		return true
-	}
-	return false
-}
-
-func convertFileEndpoints(fileEndpoints []fileEndpoint) []portainer.Endpoint {
-	convertedEndpoints := make([]portainer.Endpoint, 0)
-
-	for _, e := range fileEndpoints {
-		endpoint := portainer.Endpoint{
-			Name:      e.Name,
-			URL:       e.URL,
-			TLSConfig: portainer.TLSConfiguration{},
-		}
-		if e.TLS {
-			endpoint.TLSConfig.TLS = true
-			endpoint.TLSConfig.TLSSkipVerify = e.TLSSkipVerify
-			endpoint.TLSConfig.TLSCACertPath = e.TLSCACert
-			endpoint.TLSConfig.TLSCertPath = e.TLSCert
-			endpoint.TLSConfig.TLSKeyPath = e.TLSKey
-		}
-		convertedEndpoints = append(convertedEndpoints, endpoint)
-	}
-
-	return convertedEndpoints
-}
-
-func endpointExists(endpoint *portainer.Endpoint, endpoints []portainer.Endpoint) int {
-	for idx, v := range endpoints {
-		if endpoint.Name == v.Name && isValidEndpoint(&v) {
-			return idx
-		}
-	}
-	return -1
-}
-
-func mergeEndpointIfRequired(original, updated *portainer.Endpoint) *portainer.Endpoint {
-	var endpoint *portainer.Endpoint
-	if original.URL != updated.URL || original.TLSConfig.TLS != updated.TLSConfig.TLS ||
-		(updated.TLSConfig.TLS && original.TLSConfig.TLSSkipVerify != updated.TLSConfig.TLSSkipVerify) ||
-		(updated.TLSConfig.TLS && original.TLSConfig.TLSCACertPath != updated.TLSConfig.TLSCACertPath) ||
-		(updated.TLSConfig.TLS && original.TLSConfig.TLSCertPath != updated.TLSConfig.TLSCertPath) ||
-		(updated.TLSConfig.TLS && original.TLSConfig.TLSKeyPath != updated.TLSConfig.TLSKeyPath) {
-		endpoint = original
-		endpoint.URL = updated.URL
-		if updated.TLSConfig.TLS {
-			endpoint.TLSConfig.TLS = true
-			endpoint.TLSConfig.TLSSkipVerify = updated.TLSConfig.TLSSkipVerify
-			endpoint.TLSConfig.TLSCACertPath = updated.TLSConfig.TLSCACertPath
-			endpoint.TLSConfig.TLSCertPath = updated.TLSConfig.TLSCertPath
-			endpoint.TLSConfig.TLSKeyPath = updated.TLSConfig.TLSKeyPath
-		} else {
-			endpoint.TLSConfig.TLS = false
-			endpoint.TLSConfig.TLSSkipVerify = false
-			endpoint.TLSConfig.TLSCACertPath = ""
-			endpoint.TLSConfig.TLSCertPath = ""
-			endpoint.TLSConfig.TLSKeyPath = ""
-		}
-	}
-	return endpoint
-}
-
-func (sync synchronization) requireSync() bool {
-	if len(sync.endpointsToCreate) != 0 || len(sync.endpointsToUpdate) != 0 || len(sync.endpointsToDelete) != 0 {
-		return true
-	}
-	return false
-}
-
-func prepareSyncData(storedEndpoints, fileEndpoints []portainer.Endpoint) *synchronization {
-	endpointsToCreate := make([]*portainer.Endpoint, 0)
-	endpointsToUpdate := make([]*portainer.Endpoint, 0)
-	endpointsToDelete := make([]*portainer.Endpoint, 0)
-
-	for idx := range storedEndpoints {
-		fidx := endpointExists(&storedEndpoints[idx], fileEndpoints)
-		if fidx != -1 {
-			endpoint := mergeEndpointIfRequired(&storedEndpoints[idx], &fileEndpoints[fidx])
-			if endpoint != nil {
-				log.Printf("New definition for a stored endpoint found in file, updating database. [name: %v] [url: %v]\n", endpoint.Name, endpoint.URL)
-				endpointsToUpdate = append(endpointsToUpdate, endpoint)
-			}
-		} else {
-			log.Printf("Stored endpoint not found in file (definition might be invalid), removing from database. [name: %v] [url: %v]", storedEndpoints[idx].Name, storedEndpoints[idx].URL)
-			endpointsToDelete = append(endpointsToDelete, &storedEndpoints[idx])
-		}
-	}
-
-	for idx, endpoint := range fileEndpoints {
-		if !isValidEndpoint(&endpoint) {
-			log.Printf("Invalid file endpoint definition, skipping. [name: %v] [url: %v]", endpoint.Name, endpoint.URL)
-			continue
-		}
-		sidx := endpointExists(&fileEndpoints[idx], storedEndpoints)
-		if sidx == -1 {
-			log.Printf("File endpoint not found in database, adding to database. [name: %v] [url: %v]", fileEndpoints[idx].Name, fileEndpoints[idx].URL)
-			endpointsToCreate = append(endpointsToCreate, &fileEndpoints[idx])
-		}
-	}
-
-	return &synchronization{
-		endpointsToCreate: endpointsToCreate,
-		endpointsToUpdate: endpointsToUpdate,
-		endpointsToDelete: endpointsToDelete,
-	}
-}

api/cron/job_script_execution.go

@@ -16,17 +16,17 @@ type ScriptExecutionJobRunner struct {
 
 // ScriptExecutionJobContext represents the context of execution of a ScriptExecutionJob
 type ScriptExecutionJobContext struct {
-	jobService      portainer.JobService
-	endpointService portainer.EndpointService
-	fileService     portainer.FileService
+	dataStore   portainer.DataStore
+	jobService  portainer.JobService
+	fileService portainer.FileService
 }
 
 // NewScriptExecutionJobContext returns a new context that can be used to execute a ScriptExecutionJob
-func NewScriptExecutionJobContext(jobService portainer.JobService, endpointService portainer.EndpointService, fileService portainer.FileService) *ScriptExecutionJobContext {
+func NewScriptExecutionJobContext(jobService portainer.JobService, dataStore portainer.DataStore, fileService portainer.FileService) *ScriptExecutionJobContext {
 	return &ScriptExecutionJobContext{
-		jobService:      jobService,
-		endpointService: endpointService,
-		fileService:     fileService,
+		jobService:  jobService,
+		dataStore:   dataStore,
+		fileService: fileService,
 	}
 }
 
@@ -56,7 +56,7 @@ func (runner *ScriptExecutionJobRunner) Run() {
 
 	targets := make([]*portainer.Endpoint, 0)
 	for _, endpointID := range runner.schedule.ScriptExecutionJob.Endpoints {
-		endpoint, err := runner.context.endpointService.Endpoint(endpointID)
+		endpoint, err := runner.context.dataStore.Endpoint().Endpoint(endpointID)
 		if err != nil {
 			log.Printf("scheduled job error (script execution). Unable to retrieve information about endpoint (id=%d) (err=%s)\n", endpointID, err)
 			return

api/cron/job_snapshot.go

@@ -14,15 +14,15 @@ type SnapshotJobRunner struct {
 
 // SnapshotJobContext represents the context of execution of a SnapshotJob
 type SnapshotJobContext struct {
-	endpointService portainer.EndpointService
-	snapshotter     portainer.Snapshotter
+	dataStore   portainer.DataStore
+	snapshotter portainer.Snapshotter
 }
 
 // NewSnapshotJobContext returns a new context that can be used to execute a SnapshotJob
-func NewSnapshotJobContext(endpointService portainer.EndpointService, snapshotter portainer.Snapshotter) *SnapshotJobContext {
+func NewSnapshotJobContext(dataStore portainer.DataStore, snapshotter portainer.Snapshotter) *SnapshotJobContext {
 	return &SnapshotJobContext{
-		endpointService: endpointService,
-		snapshotter:     snapshotter,
+		dataStore:   dataStore,
+		snapshotter: snapshotter,
 	}
 }
 
@@ -46,20 +46,20 @@ func (runner *SnapshotJobRunner) GetSchedule() *portainer.Schedule {
 // retrieve the latest version of the endpoint right after a snapshot.
 func (runner *SnapshotJobRunner) Run() {
 	go func() {
-		endpoints, err := runner.context.endpointService.Endpoints()
+		endpoints, err := runner.context.dataStore.Endpoint().Endpoints()
 		if err != nil {
 			log.Printf("background schedule error (endpoint snapshot). Unable to retrieve endpoint list (err=%s)\n", err)
 			return
 		}
 
 		for _, endpoint := range endpoints {
-			if endpoint.Type == portainer.AzureEnvironment || endpoint.Type == portainer.EdgeAgentEnvironment {
+			if endpoint.Type == portainer.EdgeAgentEnvironment {
 				continue
 			}
 
 			snapshot, snapshotError := runner.context.snapshotter.CreateSnapshot(&endpoint)
 
-			latestEndpointReference, err := runner.context.endpointService.Endpoint(endpoint.ID)
+			latestEndpointReference, err := runner.context.dataStore.Endpoint().Endpoint(endpoint.ID)
 			if latestEndpointReference == nil {
 				log.Printf("background schedule error (endpoint snapshot). Endpoint not found inside the database anymore (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
 				continue
@@ -75,7 +75,7 @@ func (runner *SnapshotJobRunner) Run() {
 				latestEndpointReference.Snapshots = []portainer.Snapshot{*snapshot}
 			}
 
-			err = runner.context.endpointService.UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
+			err = runner.context.dataStore.Endpoint().UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
 			if err != nil {
 				log.Printf("background schedule error (endpoint snapshot). Unable to update endpoint (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
 				return

api/cron/job_telemetry.go

@@ -0,0 +1,649 @@
+package cron
+
+import (
+	"log"
+	"runtime"
+	"time"
+
+	"github.com/portainer/portainer/api"
+)
+
+// TelemetryJobRunner is used to run a TelemetryJob
+type TelemetryJobRunner struct {
+	schedule *portainer.Schedule
+	context  *TelemetryJobContext
+}
+
+// TelemetryJobContext represents the context of execution of a TelemetryJob
+type TelemetryJobContext struct {
+	dataStore portainer.DataStore
+}
+
+// NewTelemetryJobContext returns a new context that can be used to execute a TelemetryJob
+func NewTelemetryJobContext(dataStore portainer.DataStore) *TelemetryJobContext {
+	return &TelemetryJobContext{
+		dataStore: dataStore,
+	}
+}
+
+// NewTelemetryJobRunner returns a new runner that can be scheduled
+func NewTelemetryJobRunner(schedule *portainer.Schedule, context *TelemetryJobContext) *TelemetryJobRunner {
+	return &TelemetryJobRunner{
+		schedule: schedule,
+		context:  context,
+	}
+}
+
+// GetSchedule returns the schedule associated to the runner
+func (runner *TelemetryJobRunner) GetSchedule() *portainer.Schedule {
+	return runner.schedule
+}
+
+type (
+	TelemetryData struct {
+		Identifier      string                       `json:"Identifier"`
+		DockerHub       DockerHubTelemetryData       `json:"DockerHub"`
+		EdgeCompute     EdgeComputeTelemetryData     `json:"EdgeCompute"`
+		Endpoint        EndpointTelemetryData        `json:"Endpoint"`
+		EndpointGroup   EndpointGroupTelemetryData   `json:"EndpointGroup"`
+		Registry        RegistryTelemetryData        `json:"Registry"`
+		ResourceControl ResourceControlTelemetryData `json:"ResourceControl"`
+		Runtime         RuntimeTelemetryData         `json:"Runtime"`
+		Settings        SettingsTelemetryData        `json:"Settings"`
+		Stack           StackTelemetryData           `json:"Stack"`
+		Tag             TagTelemetryData             `json:"Tag"`
+		Team            TeamTelemetryData            `json:"Team"`
+		User            UserTelemetryData            `json:"User"`
+		Webhook         WebhookTelemetryData         `json:"Webhook"`
+
+		// TODO: Timestamp only for test purposes
+		Timestamp int64 `json:"Timestamp"`
+	}
+
+	DockerHubTelemetryData struct {
+		Authentication bool `json:"Authentication"`
+	}
+
+	EdgeComputeTelemetryData struct {
+		Schedule EdgeComputeScheduleTelemetryData `json:"Schedule"`
+	}
+
+	EdgeComputeScheduleTelemetryData struct {
+		Count     int `json:"Count"`
+		Recurring int `json:"Recurring"`
+	}
+
+	EndpointTelemetryData struct {
+		Count                   int                                `json:"Count"`
+		DockerEndpointCount     int                                `json:"DockerEndpointCount"`
+		KubernetesEndpointCount int                                `json:"KubernetesEndpointCount"`
+		Endpoints               []EndpointEnvironmentTelemetryData `json:"Endpoints"`
+
+		// TODO: revamp
+		//TLSEndpointCount      int `json:"TLSEndpointCount"`
+		//AgentEndpointCount    int `json:"AgentEndpointCount"`
+		//EdgeEndpointCount     int `json:"EdgeEndpointCount"`
+		//StandardEndpointCount int `json:"StandardEndpointCount"` // aka unsecured
+		//
+		//DockerEndpoints int      `json:"DockerEndpoints"`
+		//SwarmEndpoints  int      `json:"SwarmEndpoints"`
+		//DockerVersions  []string `json:"DockerVersions"`
+		//
+		//KubernetesEndpoints int      `json:"KubernetesEndpoints"`
+		//KubernetesVersions  []string `json:"KubernetesVersions"`
+
+		// TODO: cluster node count
+	}
+
+	EndpointEnvironmentTelemetryData struct {
+		Environment string                                     `json:"Environment"`
+		Agent       bool                                       `json:"Agent"`
+		Edge        bool                                       `json:"Edge"`
+		Docker      EndpointEnvironmentDockerTelemetryData     `json:"Docker"`
+		Kubernetes  EndpointEnvironmentKubernetesTelemetryData `json:"Kubernetes"`
+	}
+
+	EndpointEnvironmentDockerTelemetryData struct {
+		Version    string `json:"Version"`
+		Swarm      bool   `json:"Swarm"`
+		Containers int    `json:"Containers"`
+		Images     int    `json:"Images"`
+		Volumes    int    `json:"Volumes"`
+		Services   int    `json:"Services"`
+		Stacks     int    `json:"Stacks"`
+		Nodes      int    `json:"Nodes"`
+	}
+
+	EndpointEnvironmentKubernetesTelemetryData struct {
+		Version string `json:"Version"`
+		Nodes   int    `json:"Nodes"`
+	}
+
+	EndpointGroupTelemetryData struct {
+		Count int `json:"Count"`
+	}
+
+	RegistryTelemetryData struct {
+		Count int `json:"Count"`
+
+		Registries []RegistryConfigurationTelemetryData `json:"Registries"`
+
+		// TODO: revamp
+		//QuayRegistryCount   int `json:"QuayRegistryCount"`
+		//AzureRegistryCount  int `json:"AzureRegistryCount"`
+		//GitlabRegistryCount int `json:"GitlabRegistryCount"`
+		//CustomRegistryCount int `json:"CustomRegistryCount"`
+	}
+
+	RegistryConfigurationTelemetryData struct {
+		Type string `json:"Type"`
+	}
+
+	ResourceControlTelemetryData struct {
+		Count      int `json:"Count"`
+		Containers int `json:"Containers"`
+		Services   int `json:"Services"`
+		Volumes    int `json:"Volumes"`
+		Networks   int `json:"Networks"`
+		Secrets    int `json:"Secrets"`
+		Configs    int `json:"Config"`
+		Stacks     int `json:"Stacks"`
+	}
+
+	RuntimeTelemetryData struct {
+		PortainerVersion string `json:"PortainerVersion"`
+		Platform         string `json:"Platform"`
+		Arch             string `json:"Arch"`
+	}
+
+	// TODO: add EdgeCompute feature switch telemetry
+	SettingsTelemetryData struct {
+		AuthenticationMode   string                      `json:"AuthenticationMode"`
+		UseLogoURL           bool                        `json:"UseLogoURL"`
+		UseBlackListedLabels bool                        `json:"UseBlackListedLabels"`
+		Docker               SettingsDockerTelemetryData `json:"Docker"`
+		HostManagement       bool                        `json:"HostManagement"`
+		SnapshotInterval     float64                     `json:"SnapshotInterval"`
+	}
+
+	SettingsDockerTelemetryData struct {
+		RestrictBindMounts     bool `json:"RestrictBindMounts"`
+		RestrictPrivilegedMode bool `json:"RestrictPrivilegedMode"`
+		RestrictVolumeBrowser  bool `json:"RestrictVolumeBrowser"`
+	}
+
+	StackTelemetryData struct {
+		Count      int `json:"Count"`
+		Standalone int `json:"Standalone"`
+		Swarm      int `json:"Swarm"`
+	}
+
+	TagTelemetryData struct {
+		Count int `json:"Count"`
+	}
+
+	TeamTelemetryData struct {
+		Count           int `json:"Count"`
+		TeamLeaderCount int `json:"TeamLeaderCount"`
+	}
+
+	UserTelemetryData struct {
+		Count             int `json:"Count"`
+		AdminUserCount    int `json:"AdminUserCount"`
+		StandardUserCount int `json:"StandardUserCount"`
+	}
+
+	WebhookTelemetryData struct {
+		Count int `json:"Count"`
+	}
+)
+
+const AuthenticationMethodInternal = "internal"
+const AuthenticationMethodLDAP = "ldap"
+const AuthenticationMethodOAuth = "oauth"
+const EndpointEnvironmentDocker = "docker"
+const EndpointEnvironmentKubernetes = "kubernetes"
+const RegistryConfigurationTypeCustom = "custom"
+const RegistryConfigurationTypeQuay = "quay"
+const RegistryConfigurationTypeAzure = "azure"
+const RegistryConfigurationTypeGitlab = "gitlab"
+
+// Run triggers the execution of the schedule.
+// It will compute the telemetry data using the data available inside the database and send it to the telemetry server.
+func (runner *TelemetryJobRunner) Run() {
+	go func() {
+		telemetryData, err := initTelemetryData(runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to init telemetry data (err=%s)\n", err)
+			return
+		}
+
+		err = computeDockerHubTelemetry(telemetryData, runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to compute dockerhub telemetry (err=%s)\n", err)
+			return
+		}
+
+		err = computeEdgeComputeTelemetry(telemetryData, runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to compute Edge compute telemetry (err=%s)\n", err)
+			return
+		}
+
+		err = computeEndpointTelemetry(telemetryData, runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to compute endpoint telemetry (err=%s)\n", err)
+			return
+		}
+
+		err = computeEndpointGroupTelemetry(telemetryData, runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to compute endpoint group telemetry (err=%s)\n", err)
+			return
+		}
+
+		err = computeRegistryTelemetry(telemetryData, runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to compute registry telemetry (err=%s)\n", err)
+			return
+		}
+
+		err = computeResourceControlTelemetry(telemetryData, runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to compute resource control telemetry (err=%s)\n", err)
+			return
+		}
+
+		computeRuntimeTelemetry(telemetryData)
+
+		err = computeSettingsTelemetry(telemetryData, runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to compute settings telemetry (err=%s)\n", err)
+			return
+		}
+
+		err = computeStackTelemetry(telemetryData, runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to compute stack telemetry (err=%s)\n", err)
+			return
+		}
+
+		err = computeTagTelemetry(telemetryData, runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to compute tag telemetry (err=%s)\n", err)
+			return
+		}
+
+		err = computeTeamTelemetry(telemetryData, runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to compute team telemetry (err=%s)\n", err)
+			return
+		}
+
+		err = computeUserTelemetry(telemetryData, runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to compute user telemetry (err=%s)\n", err)
+			return
+		}
+
+		err = computeWebhookTelemetry(telemetryData, runner.context.dataStore)
+		if err != nil {
+			log.Printf("background schedule error (telemetry). Unable to compute webhook telemetry (err=%s)\n", err)
+			return
+		}
+	}()
+}
+
+func initTelemetryData(dataStore portainer.DataStore) (*TelemetryData, error) {
+	telemetryData := &TelemetryData{}
+
+	telemetryConfiguration, err := dataStore.Telemetry().Telemetry()
+	if err != nil {
+		return nil, err
+	}
+
+	telemetryData.Identifier = telemetryConfiguration.TelemetryID
+
+	return telemetryData, nil
+}
+
+func computeDockerHubTelemetry(telemetryData *TelemetryData, dataStore portainer.DataStore) error {
+	dockerhub, err := dataStore.DockerHub().DockerHub()
+	if err != nil {
+		return err
+	}
+
+	telemetryData.DockerHub = DockerHubTelemetryData{
+		Authentication: dockerhub.Authentication,
+	}
+
+	return nil
+}
+
+// TODO: add telemetry for Edge compute features (Edge groups, Edge stacks)
+func computeEdgeComputeTelemetry(telemetryData *TelemetryData, dataStore portainer.DataStore) error {
+	telemetryData.EdgeCompute = EdgeComputeTelemetryData{}
+
+	schedules, err := dataStore.Schedule().Schedules()
+	if err != nil {
+		return err
+	}
+
+	scheduleTelemetryData := EdgeComputeScheduleTelemetryData{
+		Count:     len(schedules),
+		Recurring: 0,
+	}
+
+	for _, schedule := range schedules {
+		if schedule.JobType == portainer.ScriptExecutionJobType && schedule.Recurring {
+			scheduleTelemetryData.Recurring++
+		}
+	}
+
+	telemetryData.EdgeCompute.Schedule = scheduleTelemetryData
+
+	return nil
+}
+
+// TODO: add telemetry for Kubernetes endpoints
+func computeEndpointTelemetry(telemetryData *TelemetryData, dataStore portainer.DataStore) error {
+	endpoints, err := dataStore.Endpoint().Endpoints()
+	if err != nil {
+		return err
+	}
+
+	endpointsTelemetry := make([]EndpointEnvironmentTelemetryData, 0)
+	dockerEndpoints, kubernetesEndpoints := 0, 0
+	for _, endpoint := range endpoints {
+		endpointTelemetry := EndpointEnvironmentTelemetryData{}
+
+		switch endpoint.Type {
+		case portainer.DockerEnvironment:
+			endpointTelemetry.Environment = EndpointEnvironmentDocker
+			endpointTelemetry.Agent = false
+			endpointTelemetry.Edge = false
+			endpointTelemetry.Docker = computeEndpointEnvironmentDockerTelemetry(&endpoint)
+			dockerEndpoints++
+		case portainer.AgentOnDockerEnvironment:
+			endpointTelemetry.Environment = EndpointEnvironmentDocker
+			endpointTelemetry.Agent = true
+			endpointTelemetry.Edge = false
+			endpointTelemetry.Docker = computeEndpointEnvironmentDockerTelemetry(&endpoint)
+			dockerEndpoints++
+		case portainer.EdgeAgentEnvironment:
+			endpointTelemetry.Environment = EndpointEnvironmentDocker
+			endpointTelemetry.Agent = true
+			endpointTelemetry.Edge = true
+			endpointTelemetry.Docker = computeEndpointEnvironmentDockerTelemetry(&endpoint)
+			dockerEndpoints++
+		default:
+			kubernetesEndpoints++
+		}
+
+		endpointsTelemetry = append(endpointsTelemetry, endpointTelemetry)
+	}
+
+	telemetryData.Endpoint = EndpointTelemetryData{
+		Count:                   len(endpoints),
+		DockerEndpointCount:     dockerEndpoints,
+		KubernetesEndpointCount: kubernetesEndpoints,
+		Endpoints:               endpointsTelemetry,
+	}
+
+	return nil
+}
+
+func computeEndpointEnvironmentDockerTelemetry(endpoint *portainer.Endpoint) EndpointEnvironmentDockerTelemetryData {
+	dockerTelemetryData := EndpointEnvironmentDockerTelemetryData{}
+
+	if len(endpoint.Snapshots) > 0 {
+		dockerTelemetryData.Version = endpoint.Snapshots[0].DockerVersion
+		dockerTelemetryData.Swarm = endpoint.Snapshots[0].Swarm
+		dockerTelemetryData.Containers = endpoint.Snapshots[0].HealthyContainerCount +
+			endpoint.Snapshots[0].RunningContainerCount +
+			endpoint.Snapshots[0].StoppedContainerCount +
+			endpoint.Snapshots[0].UnhealthyContainerCount
+		dockerTelemetryData.Images = endpoint.Snapshots[0].ImageCount
+		dockerTelemetryData.Volumes = endpoint.Snapshots[0].VolumeCount
+		dockerTelemetryData.Services = endpoint.Snapshots[0].ServiceCount
+		dockerTelemetryData.Stacks = endpoint.Snapshots[0].StackCount
+		dockerTelemetryData.Nodes = endpoint.Snapshots[0].NodeCount
+	}
+
+	return dockerTelemetryData
+}
+
+func computeEndpointGroupTelemetry(telemetryData *TelemetryData, dataStore portainer.DataStore) error {
+	endpointGroups, err := dataStore.EndpointGroup().EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	telemetryData.EndpointGroup = EndpointGroupTelemetryData{
+		Count: len(endpointGroups),
+	}
+
+	return nil
+}
+
+func computeRegistryTelemetry(telemetryData *TelemetryData, dataStore portainer.DataStore) error {
+	registries, err := dataStore.Registry().Registries()
+	if err != nil {
+		return err
+	}
+
+	registriesTelemetry := make([]RegistryConfigurationTelemetryData, 0)
+	for _, registry := range registries {
+		registryTelemetry := RegistryConfigurationTelemetryData{
+			Type: RegistryConfigurationTypeCustom,
+		}
+
+		switch registry.Type {
+		case portainer.AzureRegistry:
+			registryTelemetry.Type = RegistryConfigurationTypeAzure
+		case portainer.QuayRegistry:
+			registryTelemetry.Type = RegistryConfigurationTypeQuay
+		case portainer.GitlabRegistry:
+			registryTelemetry.Type = RegistryConfigurationTypeGitlab
+		}
+
+		registriesTelemetry = append(registriesTelemetry, registryTelemetry)
+	}
+
+	telemetryData.Registry = RegistryTelemetryData{
+		Count:      len(registries),
+		Registries: registriesTelemetry,
+	}
+
+	return nil
+}
+
+func computeResourceControlTelemetry(telemetryData *TelemetryData, dataStore portainer.DataStore) error {
+	resourceControls, err := dataStore.ResourceControl().ResourceControls()
+	if err != nil {
+		return err
+	}
+
+	telemetryData.ResourceControl = ResourceControlTelemetryData{
+		Count:      len(resourceControls),
+		Containers: 0,
+		Services:   0,
+		Volumes:    0,
+		Networks:   0,
+		Secrets:    0,
+		Configs:    0,
+		Stacks:     0,
+	}
+
+	for _, resourceControl := range resourceControls {
+		switch resourceControl.Type {
+		case portainer.ContainerResourceControl:
+			telemetryData.ResourceControl.Containers++
+		case portainer.ServiceResourceControl:
+			telemetryData.ResourceControl.Services++
+		case portainer.VolumeResourceControl:
+			telemetryData.ResourceControl.Volumes++
+		case portainer.NetworkResourceControl:
+			telemetryData.ResourceControl.Networks++
+		case portainer.SecretResourceControl:
+			telemetryData.ResourceControl.Secrets++
+		case portainer.ConfigResourceControl:
+			telemetryData.ResourceControl.Configs++
+		case portainer.StackResourceControl:
+			telemetryData.ResourceControl.Stacks++
+		}
+	}
+
+	return nil
+}
+
+func computeRuntimeTelemetry(telemetryData *TelemetryData) {
+	telemetryData.Runtime = RuntimeTelemetryData{
+		PortainerVersion: portainer.APIVersion,
+		Platform:         runtime.GOOS,
+		Arch:             runtime.GOARCH,
+	}
+}
+
+func computeSettingsTelemetry(telemetryData *TelemetryData, dataStore portainer.DataStore) error {
+	settings, err := dataStore.Settings().Settings()
+	if err != nil {
+		return err
+	}
+
+	telemetryData.Settings = SettingsTelemetryData{
+		AuthenticationMode:   AuthenticationMethodInternal,
+		UseLogoURL:           false,
+		UseBlackListedLabels: false,
+		Docker: SettingsDockerTelemetryData{
+			RestrictBindMounts:     !settings.AllowBindMountsForRegularUsers,
+			RestrictPrivilegedMode: !settings.AllowPrivilegedModeForRegularUsers,
+			RestrictVolumeBrowser:  !settings.AllowVolumeBrowserForRegularUsers,
+		},
+		HostManagement:   settings.EnableHostManagementFeatures,
+		SnapshotInterval: 0,
+	}
+
+	switch settings.AuthenticationMethod {
+	case portainer.AuthenticationLDAP:
+		telemetryData.Settings.AuthenticationMode = AuthenticationMethodLDAP
+	case portainer.AuthenticationOAuth:
+		telemetryData.Settings.AuthenticationMode = AuthenticationMethodOAuth
+	}
+
+	if settings.LogoURL != "" {
+		telemetryData.Settings.UseLogoURL = true
+	}
+
+	if len(settings.BlackListedLabels) > 0 {
+		telemetryData.Settings.UseBlackListedLabels = true
+	}
+
+	if settings.SnapshotInterval != "" {
+		duration, err := time.ParseDuration(settings.SnapshotInterval)
+		if err != nil {
+			log.Printf("background schedule warning (telemetry). Unable to parse snapshot interval duration (err=%s)\n", err)
+		} else {
+			telemetryData.Settings.SnapshotInterval = duration.Seconds()
+		}
+	}
+
+	return nil
+}
+
+func computeStackTelemetry(telemetryData *TelemetryData, dataStore portainer.DataStore) error {
+	stacks, err := dataStore.Stack().Stacks()
+	if err != nil {
+		return err
+	}
+
+	telemetryData.Stack = StackTelemetryData{
+		Count:      len(stacks),
+		Standalone: 0,
+		Swarm:      0,
+	}
+
+	for _, stack := range stacks {
+		if stack.Type == portainer.DockerComposeStack {
+			telemetryData.Stack.Standalone++
+		} else {
+			telemetryData.Stack.Swarm++
+		}
+	}
+
+	return nil
+}
+
+func computeTagTelemetry(telemetryData *TelemetryData, dataStore portainer.DataStore) error {
+	tags, err := dataStore.Tag().Tags()
+	if err != nil {
+		return err
+	}
+
+	telemetryData.Tag = TagTelemetryData{
+		Count: len(tags),
+	}
+
+	return nil
+}
+
+func computeTeamTelemetry(telemetryData *TelemetryData, dataStore portainer.DataStore) error {
+	teams, err := dataStore.Team().Teams()
+	if err != nil {
+		return err
+	}
+
+	telemetryData.Team = TeamTelemetryData{
+		Count:           len(teams),
+		TeamLeaderCount: 0,
+	}
+
+	teamMemberships, err := dataStore.TeamMembership().TeamMemberships()
+	if err != nil {
+		return err
+	}
+
+	for _, membership := range teamMemberships {
+		if membership.Role == portainer.TeamLeader {
+			telemetryData.Team.TeamLeaderCount++
+		}
+	}
+
+	return nil
+}
+
+func computeUserTelemetry(telemetryData *TelemetryData, dataStore portainer.DataStore) error {
+	users, err := dataStore.User().Users()
+	if err != nil {
+		return err
+	}
+
+	telemetryData.User = UserTelemetryData{
+		Count:             len(users),
+		AdminUserCount:    0,
+		StandardUserCount: 0,
+	}
+
+	for _, user := range users {
+		if user.Role == portainer.AdministratorRole {
+			telemetryData.User.AdminUserCount++
+		} else {
+			telemetryData.User.StandardUserCount++
+		}
+	}
+
+	return nil
+}
+
+func computeWebhookTelemetry(telemetryData *TelemetryData, store *bolt.Store) error {
+	webhooks, err := store.WebhookService.Webhooks()
+	if err != nil {
+		return err
+	}
+
+	telemetryData.Webhook = WebhookTelemetryData{
+		Count: len(webhooks),
+	}
+
+	return nil
+}

api/cron/job_telemetry_test.go

@@ -0,0 +1,418 @@
+package cron
+
+import (
+	"encoding/json"
+	"log"
+	"math"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/gofrs/uuid"
+)
+
+// max number of instances
+const instanceCount = 1500
+
+// min date for initial report
+var minInitialReportDate = time.Date(2020, 3, 0, 0, 0, 0, 0, time.UTC).Unix()
+
+// max date for initial report
+var maxInitialReportDate = time.Date(2020, 4, 0, 0, 0, 0, 0, time.UTC).Unix()
+
+// edge compute
+const edgeComputeMaxScheduleCount = 10
+
+// endpoint
+const endpointMaxCount = 50
+const dockerContainerMaxCount = 250
+const dockerImageMaxCount = 1000
+const dockerVolumeMaxCount = 250
+const dockerServiceMaxCount = 100
+const dockerStackMaxCount = 30
+const dockerNodeMaxCount = 9
+const kubernetesNodeMaxCount = 15
+
+// endpointgroup
+const endpointGroupMaxCount = 25
+
+// registry
+const registryMaxCount = 10
+
+// resourceControl
+const resourceControlTypeMaxCount = 250
+
+// settings
+const snapshotIntervalMinValue = 5     // in seconds
+const snapshotIntervalMaxValue = 86400 // in seconds == 24h
+
+// stack
+const stackMaxCount = 150
+
+// tag
+const tagMaxCount = 50
+
+// team
+const teamMaxCount = 20
+const teamLeaderMaxCount = 7
+
+// user
+const userAdminMaxCount = 10
+const userStandardMaxCount = 150
+
+// webhooks
+const webhookMaxCount = 25
+
+var dockerVersions = []string{
+	"18.03.1",
+	"18.03.2",
+	"18.03.3",
+	"18.09.1",
+	"18.09.2",
+	"18.09.3",
+	"19.03.1",
+	"19.03.2",
+	"19.03.3",
+}
+
+var kubernetesVersions = []string{
+	"v1.15.1",
+	"v1.16.1",
+	"v1.16.2",
+	"v1.17.1",
+	"v1.17.2",
+	"v1.18.1",
+	"v1.18.2",
+}
+
+var registryTypes = []string{
+	RegistryConfigurationTypeCustom,
+	RegistryConfigurationTypeGitlab,
+	RegistryConfigurationTypeQuay,
+	RegistryConfigurationTypeAzure,
+}
+
+var portainerVersions = []string{
+	"2.0.0",
+	"2.0.1",
+	"2.1.0",
+	"2.1.1",
+	"2.2.0",
+}
+
+var platforms = []string{
+	"linux",
+	"windows",
+}
+
+var archs = []string{
+	"amd64",
+	"arm",
+	"arm64",
+}
+
+var authenticationModes = []string{
+	AuthenticationMethodOAuth,
+	AuthenticationMethodLDAP,
+	AuthenticationMethodInternal,
+}
+
+func prettyPrint(i interface{}) string {
+	s, _ := json.MarshalIndent(i, "", "\t")
+	return string(s)
+}
+
+// need timestamp (random between two dates say 3month period with weekly telemetry report)
+// 3 month period, weekly report = 3 * 4 = 12 reports per instance
+// generate a random number of reports per instance between 1 and 12
+// define a total number of instances = 3000 (3000 unique UUIDs)
+// this random generator will generate between 3000 * 1 and 3000 * 12 reports
+
+const fileName = "reports.json"
+
+func TestGenerator(t *testing.T) {
+
+	//rand.Seed(time.Now().UnixNano())
+
+	file, err := os.OpenFile(fileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm)
+	if err != nil {
+		t.Fatalf("an error occured: %s", err)
+	}
+	defer file.Close()
+
+	for i := 0; i < instanceCount; i++ {
+		token, err := uuid.NewV4()
+		if err != nil {
+			t.Fatalf("an error occured: %s", err)
+		}
+
+		instanceID := token.String()
+
+		reportsPerInstance := rand.Intn(30) + 1
+		log.Printf("Instance %s (#%d): generating %d reports", instanceID, i, reportsPerInstance)
+
+		reportDateTime := utilsRandDateTime(minInitialReportDate, maxInitialReportDate)
+
+		for j := 0; j < reportsPerInstance; j++ {
+			report := generateRandomTelemetryData(instanceID, reportDateTime.Unix())
+
+			//log.Printf("Report: %s", prettyPrint(report))
+
+			data, err := json.Marshal(report)
+			if err != nil {
+				t.Fatalf("an error occured: %s", err)
+			}
+
+			_, err = file.Write(data)
+			if err != nil {
+				t.Fatalf("an error occured: %s", err)
+			}
+
+			_, err = file.WriteString("\n")
+			if err != nil {
+				t.Fatalf("an error occured: %s", err)
+			}
+
+			reportDateTime = reportDateTime.AddDate(0, 0, 1)
+		}
+	}
+}
+
+func utilsRandBool() bool {
+	return rand.Int()%2 == 0
+}
+
+func utilsRandChoice(choices []string) string {
+	randomIndex := rand.Intn(len(choices))
+	return choices[randomIndex]
+}
+
+func utilsRandFloat(min, max float64) float64 {
+	randFloat := min + rand.Float64()*(max-min)
+	return math.Round(randFloat*100) / 100
+}
+
+func utilsRandDateTime(min, max int64) time.Time {
+	delta := max - min
+	sec := rand.Int63n(delta) + min
+	return time.Unix(sec, 0)
+}
+
+func generateRandomTelemetryData(instanceID string, timestamp int64) *TelemetryData {
+	telemetryData := &TelemetryData{
+		Identifier:      instanceID,
+		Timestamp:       timestamp,
+		DockerHub:       randomDockerHubTelemetryData(),
+		EdgeCompute:     randomEdgeComputeTelemetryData(),
+		Endpoint:        randomEndpointTelemetryData(),
+		EndpointGroup:   randomEndpointGroupTelemetryData(),
+		Registry:        randomRegistryTelemetryData(),
+		ResourceControl: randomResourceControlTelemetryData(),
+		Runtime:         randomRuntimeTelemetryData(),
+		Settings:        randomSettingsTelemetryData(),
+		Stack:           randomStackTelemetryData(),
+		Tag:             randomTagTelemetryData(),
+		Team:            randomTeamTelemetryData(),
+		User:            randomUserTelemetryData(),
+		Webhook:         randomWebhookTelemetryData(),
+	}
+
+	return telemetryData
+}
+
+func randomDockerHubTelemetryData() DockerHubTelemetryData {
+	return DockerHubTelemetryData{
+		Authentication: utilsRandBool(),
+	}
+}
+
+func randomEdgeComputeTelemetryData() EdgeComputeTelemetryData {
+	scheduleCount := rand.Intn(edgeComputeMaxScheduleCount)
+	recurringCount := 0
+	if scheduleCount != 0 {
+		recurringCount = rand.Intn(scheduleCount)
+	}
+
+	return EdgeComputeTelemetryData{
+		Schedule: EdgeComputeScheduleTelemetryData{
+			Count:     scheduleCount,
+			Recurring: recurringCount,
+		},
+	}
+}
+
+func randomEndpointTelemetryData() EndpointTelemetryData {
+	endpointCount := rand.Intn(endpointMaxCount)
+
+	endpoints := make([]EndpointEnvironmentTelemetryData, 0)
+	dockerEndpoints, KubernetesEndpoints := 0, 0
+	for i := 0; i < endpointCount; i++ {
+		endpointEnvTelemetry, dockerEnv := randomEndpointEnvironmentTelemetryData()
+
+		if dockerEnv {
+			dockerEndpoints++
+		} else {
+			KubernetesEndpoints++
+		}
+
+		endpoints = append(endpoints, endpointEnvTelemetry)
+	}
+
+	return EndpointTelemetryData{
+		Count:                   endpointCount,
+		Endpoints:               endpoints,
+		DockerEndpointCount:     dockerEndpoints,
+		KubernetesEndpointCount: KubernetesEndpoints,
+	}
+}
+
+func randomEndpointEnvironmentTelemetryData() (EndpointEnvironmentTelemetryData, bool) {
+	endpointEnvTelemetry := EndpointEnvironmentTelemetryData{
+		Environment: EndpointEnvironmentDocker,
+		Agent:       utilsRandBool(),
+		Edge:        false,
+	}
+
+	dockerEnvironment := true
+
+	if utilsRandBool() {
+		endpointEnvTelemetry.Environment = EndpointEnvironmentKubernetes
+	}
+
+	if endpointEnvTelemetry.Agent {
+		endpointEnvTelemetry.Edge = utilsRandBool()
+	}
+
+	if endpointEnvTelemetry.Environment == EndpointEnvironmentDocker {
+		endpointEnvTelemetry.Docker = EndpointEnvironmentDockerTelemetryData{
+			Version:    utilsRandChoice(dockerVersions),
+			Swarm:      false,
+			Containers: rand.Intn(dockerContainerMaxCount),
+			Images:     rand.Intn(dockerImageMaxCount),
+			Volumes:    rand.Intn(dockerVolumeMaxCount),
+			Services:   0,
+			Stacks:     rand.Intn(dockerStackMaxCount),
+			Nodes:      1,
+		}
+
+		if utilsRandBool() {
+			endpointEnvTelemetry.Docker.Swarm = true
+			endpointEnvTelemetry.Docker.Services = rand.Intn(dockerServiceMaxCount)
+			endpointEnvTelemetry.Docker.Nodes = rand.Intn(dockerNodeMaxCount) + 1
+		}
+
+	} else {
+		dockerEnvironment = false
+		endpointEnvTelemetry.Kubernetes = EndpointEnvironmentKubernetesTelemetryData{
+			Version: utilsRandChoice(kubernetesVersions),
+			Nodes:   rand.Intn(kubernetesNodeMaxCount) + 1,
+		}
+	}
+
+	return endpointEnvTelemetry, dockerEnvironment
+}
+
+func randomEndpointGroupTelemetryData() EndpointGroupTelemetryData {
+	return EndpointGroupTelemetryData{
+		Count: rand.Intn(endpointGroupMaxCount),
+	}
+}
+
+func randomRegistryTelemetryData() RegistryTelemetryData {
+	registryCount := rand.Intn(registryMaxCount)
+
+	registries := make([]RegistryConfigurationTelemetryData, 0)
+	for i := 0; i < registryCount; i++ {
+
+		registryConfTelemetry := RegistryConfigurationTelemetryData{
+			Type: utilsRandChoice(registryTypes),
+		}
+		registries = append(registries, registryConfTelemetry)
+	}
+
+	return RegistryTelemetryData{
+		Count:      registryCount,
+		Registries: registries,
+	}
+}
+
+func randomResourceControlTelemetryData() ResourceControlTelemetryData {
+	rcTelemetry := ResourceControlTelemetryData{
+		Containers: rand.Intn(resourceControlTypeMaxCount),
+		Services:   rand.Intn(resourceControlTypeMaxCount),
+		Volumes:    rand.Intn(resourceControlTypeMaxCount),
+		Networks:   rand.Intn(resourceControlTypeMaxCount),
+		Secrets:    rand.Intn(resourceControlTypeMaxCount),
+		Configs:    rand.Intn(resourceControlTypeMaxCount),
+		Stacks:     rand.Intn(resourceControlTypeMaxCount),
+	}
+
+	rcTelemetry.Count = rcTelemetry.Containers + rcTelemetry.Services + rcTelemetry.Volumes + rcTelemetry.Networks + rcTelemetry.Secrets + rcTelemetry.Configs + rcTelemetry.Stacks
+
+	return rcTelemetry
+}
+
+func randomRuntimeTelemetryData() RuntimeTelemetryData {
+	return RuntimeTelemetryData{
+		PortainerVersion: utilsRandChoice(portainerVersions),
+		Platform:         utilsRandChoice(platforms),
+		Arch:             utilsRandChoice(archs),
+	}
+}
+
+func randomSettingsTelemetryData() SettingsTelemetryData {
+	return SettingsTelemetryData{
+		AuthenticationMode:   utilsRandChoice(authenticationModes),
+		UseLogoURL:           utilsRandBool(),
+		UseBlackListedLabels: utilsRandBool(),
+		Docker: SettingsDockerTelemetryData{
+			RestrictBindMounts:     utilsRandBool(),
+			RestrictPrivilegedMode: utilsRandBool(),
+			RestrictVolumeBrowser:  utilsRandBool(),
+		},
+		HostManagement:   utilsRandBool(),
+		SnapshotInterval: utilsRandFloat(snapshotIntervalMinValue, snapshotIntervalMaxValue),
+	}
+}
+
+func randomStackTelemetryData() StackTelemetryData {
+	stackTelemetry := StackTelemetryData{
+		Standalone: rand.Intn(stackMaxCount),
+		Swarm:      rand.Intn(stackMaxCount),
+	}
+
+	stackTelemetry.Count = stackTelemetry.Standalone + stackTelemetry.Swarm
+
+	return stackTelemetry
+}
+
+func randomTagTelemetryData() TagTelemetryData {
+	return TagTelemetryData{
+		Count: rand.Intn(tagMaxCount),
+	}
+}
+
+func randomTeamTelemetryData() TeamTelemetryData {
+	return TeamTelemetryData{
+		Count:           rand.Intn(teamMaxCount),
+		TeamLeaderCount: rand.Intn(teamLeaderMaxCount),
+	}
+}
+
+func randomUserTelemetryData() UserTelemetryData {
+	ut := UserTelemetryData{
+		AdminUserCount:    rand.Intn(userAdminMaxCount),
+		StandardUserCount: rand.Intn(userStandardMaxCount),
+	}
+
+	ut.Count = ut.StandardUserCount + ut.AdminUserCount
+
+	return ut
+}
+
+func randomWebhookTelemetryData() WebhookTelemetryData {
+	return WebhookTelemetryData{
+		Count: rand.Intn(webhookMaxCount),
+	}
+}

api/docker/client.go

@@ -35,9 +35,7 @@ func NewClientFactory(signatureService portainer.DigitalSignatureService, revers
 // a specific endpoint configuration. The nodeName parameter can be used
 // with an agent enabled endpoint to target a specific node in an agent cluster.
 func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeName string) (*client.Client, error) {
-	if endpoint.Type == portainer.AzureEnvironment {
-		return nil, unsupportedEnvironmentType
-	} else if endpoint.Type == portainer.AgentOnDockerEnvironment {
+	if endpoint.Type == portainer.AgentOnDockerEnvironment {
 		return createAgentClient(endpoint, factory.signatureService, nodeName)
 	} else if endpoint.Type == portainer.EdgeAgentEnvironment {
 		return createEdgeClient(endpoint, factory.reverseTunnelService, nodeName)

api/docker/snapshot.go

@@ -20,6 +20,7 @@ func snapshot(cli *client.Client, endpoint *portainer.Endpoint) (*portainer.Snap
 
 	snapshot := &portainer.Snapshot{
 		StackCount: 0,
+		NodeCount:  1,
 	}
 
 	err = snapshotInfo(snapshot, cli)
@@ -95,6 +96,7 @@ func snapshotNodes(snapshot *portainer.Snapshot, cli *client.Client) error {
 	}
 	snapshot.TotalCPU = int(nanoCpus / 1e9)
 	snapshot.TotalMemory = totalMem
+	snapshot.NodeCount = len(nodes)
 	return nil
 }
 

api/errors.go

@@ -39,11 +39,6 @@ const (
 	ErrEndpointAccessDenied = Error("Access denied to endpoint")
 )
 
-// Azure environment errors
-const (
-	ErrAzureInvalidCredentials = Error("Invalid Azure credentials")
-)
-
 // Endpoint group errors.
 const (
 	ErrCannotRemoveDefaultGroup = Error("Cannot remove the default endpoint group")

api/exec/extension.go

@@ -34,17 +34,17 @@ var extensionBinaryMap = map[portainer.ExtensionID]string{
 // ExtensionManager represents a service used to
 // manage extension processes.
 type ExtensionManager struct {
-	processes        cmap.ConcurrentMap
-	fileService      portainer.FileService
-	extensionService portainer.ExtensionService
+	processes   cmap.ConcurrentMap
+	fileService portainer.FileService
+	dataStore   portainer.DataStore
 }
 
 // NewExtensionManager returns a pointer to an ExtensionManager
-func NewExtensionManager(fileService portainer.FileService, extensionService portainer.ExtensionService) *ExtensionManager {
+func NewExtensionManager(fileService portainer.FileService, dataStore portainer.DataStore) *ExtensionManager {
 	return &ExtensionManager{
-		processes:        cmap.New(),
-		fileService:      fileService,
-		extensionService: extensionService,
+		processes:   cmap.New(),
+		fileService: fileService,
+		dataStore:   dataStore,
 	}
 }
 
@@ -188,7 +188,7 @@ func (manager *ExtensionManager) DisableExtension(extension *portainer.Extension
 // The purpose of this function is to be ran at startup, as such most of the error handling won't block the program execution
 // and will log warning messages instead.
 func (manager *ExtensionManager) StartExtensions() error {
-	extensions, err := manager.extensionService.Extensions()
+	extensions, err := manager.dataStore.Extension().Extensions()
 	if err != nil {
 		return err
 	}
@@ -224,7 +224,7 @@ func (manager *ExtensionManager) updateAndStartExtensions(extensions []portainer
 					}
 				}
 
-				err := manager.extensionService.Persist(&extension)
+				err := manager.dataStore.Extension().Persist(&extension)
 				if err != nil {
 					return err
 				}

api/go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/docker/cli v0.0.0-20191126203649-54d085b857e9
 	github.com/docker/docker v0.0.0-00010101000000-000000000000
 	github.com/g07cha/defender v0.0.0-20180505193036-5665c627c814
+	github.com/go-ldap/ldap/v3 v3.1.8
 	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/securecookie v1.1.1
@@ -30,11 +31,7 @@ require (
 	github.com/robfig/cron/v3 v3.0.0
 	golang.org/x/crypto v0.0.0-20191128160524-b544559bb6d1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
-	gopkg.in/asn1-ber.v1 v1.0.0-00010101000000-000000000000 // indirect
-	gopkg.in/ldap.v2 v2.5.1
 	gopkg.in/src-d/go-git.v4 v4.13.1
 )
 
 replace github.com/docker/docker => github.com/docker/engine v1.4.2-0.20200204220554-5f6d6f3f2203
-
-replace gopkg.in/asn1-ber.v1 => github.com/go-asn1-ber/asn1-ber v1.3.1

api/go.sum

@@ -50,12 +50,8 @@ github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BU
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
-github.com/docker/engine v1.4.2-0.20191127222017-3152f9436292 h1:qQ7mw+CVWpRj5DWBL4CVHtBbGQdlPCj4j1evDh0ethw=
-github.com/docker/engine v1.4.2-0.20191127222017-3152f9436292/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY=
 github.com/docker/engine v1.4.2-0.20200204220554-5f6d6f3f2203 h1:QeBh8wW8pIZKlXxlMOQ8hSCMdJA+2Z/bD/iDyCAS8XU=
 github.com/docker/engine v1.4.2-0.20200204220554-5f6d6f3f2203/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY=
-github.com/docker/engine v1.13.1 h1:Cks33UT9YBW5Xyc3MtGDq2IPgqfJtJ+qkFaxc2b0Euc=
-github.com/docker/engine v1.13.1/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY=
 github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF+n1M6o=
 github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 h1:X0fj836zx99zFu83v/M79DuBn84IL/Syx1SY6Y5ZEMA=
@@ -77,6 +73,8 @@ github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aev
 github.com/go-asn1-ber/asn1-ber v1.3.1 h1:gvPdv/Hr++TRFCl0UbPFHC54P9N9jgsRPnmnr419Uck=
 github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-ldap/ldap/v3 v3.1.8 h1:5vU/2jOh9HqprwXp8aF915s9p6Z8wmbSEVF7/gdTFhM=
+github.com/go-ldap/ldap/v3 v3.1.8/go.mod h1:5Zun81jBTabRaI8lzN7E1JjyEl1g6zI6u9pd8luAK4Q=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -140,8 +138,6 @@ github.com/mattn/go-shellwords v1.0.6 h1:9Jok5pILi5S1MnDirGVTufYGtksUs/V2BWUP3Zk
 github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/microsoft/go-winio v0.4.8 h1:N4SmTFXUK7/jnn/UG/gm2mrHiYu9LVGvtsvULyody/c=
-github.com/microsoft/go-winio v0.4.8/go.mod h1:kcIxxtKZE55DEncT/EOvFiygPobhUWpSDqDb47poQOU=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
@@ -175,6 +171,7 @@ github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2 h1:0PfgGLys9yH
 github.com/portainer/libcrypto v0.0.0-20190723020515-23ebe86ab2c2/go.mod h1:/wIeGwJOMYc1JplE/OvYMO5korce39HddIfI8VKGyAM=
 github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33 h1:H8HR2dHdBf8HANSkUyVw4o8+4tegGcd+zyKZ3e599II=
 github.com/portainer/libhttp v0.0.0-20190806161843-ba068f58be33/go.mod h1:Y2TfgviWI4rT2qaOTHr+hq6MdKIE5YjgQAu7qwptTV0=
+github.com/portainer/portainer v0.10.1 h1:I8K345CjGWfUGsVA8c8/gqamwLCC6CIAjxZXSklAFq0=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
@@ -247,8 +244,6 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0=
 golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c h1:jceGD5YNJGgGMkJz79agzOln1K9TaZUjv5ird16qniQ=
-golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -268,8 +263,6 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/ldap.v2 v2.5.1 h1:wiu0okdNfjlBzg6UWvd1Hn8Y+Ux17/u/4nlk4CQr6tU=
-gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk=
 gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
 gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
 gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg=

api/http/client/client.go

@@ -2,12 +2,9 @@ package client
 
 import (
 	"crypto/tls"
-	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"log"
 	"net/http"
-	"net/url"
 	"strings"
 	"time"
 
@@ -19,55 +16,6 @@ const (
 	defaultHTTPTimeout       = 5
 )
 
-// HTTPClient represents a client to send HTTP requests.
-type HTTPClient struct {
-	*http.Client
-}
-
-// NewHTTPClient is used to build a new HTTPClient.
-func NewHTTPClient() *HTTPClient {
-	return &HTTPClient{
-		&http.Client{
-			Timeout: time.Second * time.Duration(defaultHTTPTimeout),
-		},
-	}
-}
-
-// AzureAuthenticationResponse represents an Azure API authentication response.
-type AzureAuthenticationResponse struct {
-	AccessToken string `json:"access_token"`
-	ExpiresOn   string `json:"expires_on"`
-}
-
-// ExecuteAzureAuthenticationRequest is used to execute an authentication request
-// against the Azure API. It re-uses the same http.Client.
-func (client *HTTPClient) ExecuteAzureAuthenticationRequest(credentials *portainer.AzureCredentials) (*AzureAuthenticationResponse, error) {
-	loginURL := fmt.Sprintf("https://login.microsoftonline.com/%s/oauth2/token", credentials.TenantID)
-	params := url.Values{
-		"grant_type":    {"client_credentials"},
-		"client_id":     {credentials.ApplicationID},
-		"client_secret": {credentials.AuthenticationKey},
-		"resource":      {"https://management.azure.com/"},
-	}
-
-	response, err := client.PostForm(loginURL, params)
-	if err != nil {
-		return nil, err
-	}
-
-	if response.StatusCode != http.StatusOK {
-		return nil, portainer.ErrAzureInvalidCredentials
-	}
-
-	var token AzureAuthenticationResponse
-	err = json.NewDecoder(response.Body).Decode(&token)
-	if err != nil {
-		return nil, err
-	}
-
-	return &token, nil
-}
-
 // Get executes a simple HTTP GET to the specified URL and returns
 // the content of the response body. Timeout can be specified via the timeout parameter,
 // will default to defaultHTTPTimeout if set to 0.

api/http/handler/auth/authenticate.go

@@ -42,12 +42,12 @@ func (handler *Handler) authenticate(w http.ResponseWriter, r *http.Request) *ht
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	settings, err := handler.SettingsService.Settings()
+	settings, err := handler.DataStore.Settings().Settings()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
 	}
 
-	u, err := handler.UserService.UserByUsername(payload.Username)
+	u, err := handler.DataStore.User().UserByUsername(payload.Username)
 	if err != nil && err != portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a user with the specified username from the database", err}
 	}
@@ -108,7 +108,7 @@ func (handler *Handler) authenticateLDAPAndCreateUser(w http.ResponseWriter, use
 		PortainerAuthorizations: portainer.DefaultPortainerAuthorizations(),
 	}
 
-	err = handler.UserService.CreateUser(user)
+	err = handler.DataStore.User().CreateUser(user)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist user inside the database", err}
 	}
@@ -146,7 +146,7 @@ func (handler *Handler) persistAndWriteToken(w http.ResponseWriter, tokenData *p
 }
 
 func (handler *Handler) addUserIntoTeams(user *portainer.User, settings *portainer.LDAPSettings) error {
-	teams, err := handler.TeamService.Teams()
+	teams, err := handler.DataStore.Team().Teams()
 	if err != nil {
 		return err
 	}
@@ -156,7 +156,7 @@ func (handler *Handler) addUserIntoTeams(user *portainer.User, settings *portain
 		return err
 	}
 
-	userMemberships, err := handler.TeamMembershipService.TeamMembershipsByUserID(user.ID)
+	userMemberships, err := handler.DataStore.TeamMembership().TeamMembershipsByUserID(user.ID)
 	if err != nil {
 		return err
 	}
@@ -174,7 +174,7 @@ func (handler *Handler) addUserIntoTeams(user *portainer.User, settings *portain
 				Role:   portainer.TeamMember,
 			}
 
-			err := handler.TeamMembershipService.CreateTeamMembership(membership)
+			err := handler.DataStore.TeamMembership().CreateTeamMembership(membership)
 			if err != nil {
 				return err
 			}

api/http/handler/auth/authenticate_oauth.go

@@ -78,7 +78,7 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	settings, err := handler.SettingsService.Settings()
+	settings, err := handler.DataStore.Settings().Settings()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
 	}
@@ -87,7 +87,7 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 		return &httperror.HandlerError{http.StatusForbidden, "OAuth authentication is not enabled", portainer.Error("OAuth authentication is not enabled")}
 	}
 
-	extension, err := handler.ExtensionService.Extension(portainer.OAuthAuthenticationExtension)
+	extension, err := handler.DataStore.Extension().Extension(portainer.OAuthAuthenticationExtension)
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Oauth authentication extension is not enabled", err}
 	} else if err != nil {
@@ -100,7 +100,7 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate through OAuth", portainer.ErrUnauthorized}
 	}
 
-	user, err := handler.UserService.UserByUsername(username)
+	user, err := handler.DataStore.User().UserByUsername(username)
 	if err != nil && err != portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve a user with the specified username from the database", err}
 	}
@@ -116,7 +116,7 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 			PortainerAuthorizations: portainer.DefaultPortainerAuthorizations(),
 		}
 
-		err = handler.UserService.CreateUser(user)
+		err = handler.DataStore.User().CreateUser(user)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist user inside the database", err}
 		}
@@ -128,7 +128,7 @@ func (handler *Handler) validateOAuth(w http.ResponseWriter, r *http.Request) *h
 				Role:   portainer.TeamMember,
 			}
 
-			err = handler.TeamMembershipService.CreateTeamMembership(membership)
+			err = handler.DataStore.TeamMembership().CreateTeamMembership(membership)
 			if err != nil {
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist team membership inside the database", err}
 			}

api/http/handler/auth/handler.go

@@ -11,8 +11,6 @@ import (
 )
 
 const (
-	// ErrInvalidCredentials is an error raised when credentials for a user are invalid
-	ErrInvalidCredentials = portainer.Error("Invalid credentials")
 	// ErrAuthDisabled is an error raised when trying to access the authentication endpoints
 	// when the server has been started with the --no-auth flag
 	ErrAuthDisabled = portainer.Error("Authentication is disabled")
@@ -21,20 +19,13 @@ const (
 // Handler is the HTTP handler used to handle authentication operations.
 type Handler struct {
 	*mux.Router
-	authDisabled          bool
-	UserService           portainer.UserService
-	CryptoService         portainer.CryptoService
-	JWTService            portainer.JWTService
-	LDAPService           portainer.LDAPService
-	SettingsService       portainer.SettingsService
-	TeamService           portainer.TeamService
-	TeamMembershipService portainer.TeamMembershipService
-	ExtensionService      portainer.ExtensionService
-	EndpointService       portainer.EndpointService
-	EndpointGroupService  portainer.EndpointGroupService
-	RoleService           portainer.RoleService
-	ProxyManager          *proxy.Manager
-	AuthorizationService  *portainer.AuthorizationService
+	authDisabled         bool
+	DataStore            portainer.DataStore
+	CryptoService        portainer.CryptoService
+	JWTService           portainer.JWTService
+	LDAPService          portainer.LDAPService
+	ProxyManager         *proxy.Manager
+	AuthorizationService *portainer.AuthorizationService
 }
 
 // NewHandler creates a handler to manage authentication operations.

api/http/handler/dockerhub/dockerhub_inspect.go

@@ -9,7 +9,7 @@ import (
 
 // GET request on /api/dockerhub
 func (handler *Handler) dockerhubInspect(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	dockerhub, err := handler.DockerHubService.DockerHub()
+	dockerhub, err := handler.DataStore.DockerHub().DockerHub()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve DockerHub details from the database", err}
 	}

api/http/handler/dockerhub/dockerhub_update.go

@@ -43,7 +43,7 @@ func (handler *Handler) dockerhubUpdate(w http.ResponseWriter, r *http.Request)
 		dockerhub.Password = payload.Password
 	}
 
-	err = handler.DockerHubService.UpdateDockerHub(dockerhub)
+	err = handler.DataStore.DockerHub().UpdateDockerHub(dockerhub)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the Dockerhub changes inside the database", err}
 	}

api/http/handler/dockerhub/handler.go

@@ -16,7 +16,7 @@ func hideFields(dockerHub *portainer.DockerHub) {
 // Handler is the HTTP handler used to handle DockerHub operations.
 type Handler struct {
 	*mux.Router
-	DockerHubService portainer.DockerHubService
+	DataStore portainer.DataStore
 }
 
 // NewHandler creates a handler to manage Dockerhub operations.

api/http/handler/edgegroups/associated_endpoints.go

@@ -11,7 +11,7 @@ func (handler *Handler) getEndpointsByTags(tagIDs []portainer.TagID, partialMatc
 		return []portainer.EndpointID{}, nil
 	}
 
-	endpoints, err := handler.EndpointService.Endpoints()
+	endpoints, err := handler.DataStore.Endpoint().Endpoints()
 	if err != nil {
 		return nil, err
 	}
@@ -20,7 +20,7 @@ func (handler *Handler) getEndpointsByTags(tagIDs []portainer.TagID, partialMatc
 
 	tags := []portainer.Tag{}
 	for _, tagID := range tagIDs {
-		tag, err := handler.TagService.Tag(tagID)
+		tag, err := handler.DataStore.Tag().Tag(tagID)
 		if err != nil {
 			return nil, err
 		}

api/http/handler/edgegroups/edgegroup_create.go

@@ -38,7 +38,7 @@ func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	edgeGroups, err := handler.EdgeGroupService.EdgeGroups()
+	edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve Edge groups from the database", err}
 	}
@@ -62,7 +62,7 @@ func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request)
 	} else {
 		endpointIDs := []portainer.EndpointID{}
 		for _, endpointID := range payload.Endpoints {
-			endpoint, err := handler.EndpointService.Endpoint(endpointID)
+			endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
 			if err != nil {
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint from the database", err}
 			}
@@ -74,7 +74,7 @@ func (handler *Handler) edgeGroupCreate(w http.ResponseWriter, r *http.Request)
 		edgeGroup.Endpoints = endpointIDs
 	}
 
-	err = handler.EdgeGroupService.CreateEdgeGroup(edgeGroup)
+	err = handler.DataStore.EdgeGroup().CreateEdgeGroup(edgeGroup)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the Edge group inside the database", err}
 	}

api/http/handler/edgegroups/edgegroup_delete.go

@@ -15,14 +15,14 @@ func (handler *Handler) edgeGroupDelete(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid Edge group identifier route variable", err}
 	}
 
-	_, err = handler.EdgeGroupService.EdgeGroup(portainer.EdgeGroupID(edgeGroupID))
+	_, err = handler.DataStore.EdgeGroup().EdgeGroup(portainer.EdgeGroupID(edgeGroupID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an Edge group with the specified identifier inside the database", err}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an Edge group with the specified identifier inside the database", err}
 	}
 
-	edgeStacks, err := handler.EdgeStackService.EdgeStacks()
+	edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve Edge stacks from the database", err}
 	}
@@ -35,7 +35,7 @@ func (handler *Handler) edgeGroupDelete(w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	err = handler.EdgeGroupService.DeleteEdgeGroup(portainer.EdgeGroupID(edgeGroupID))
+	err = handler.DataStore.EdgeGroup().DeleteEdgeGroup(portainer.EdgeGroupID(edgeGroupID))
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the Edge group from the database", err}
 	}

api/http/handler/edgegroups/edgegroup_inspect.go

@@ -15,7 +15,7 @@ func (handler *Handler) edgeGroupInspect(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid Edge group identifier route variable", err}
 	}
 
-	edgeGroup, err := handler.EdgeGroupService.EdgeGroup(portainer.EdgeGroupID(edgeGroupID))
+	edgeGroup, err := handler.DataStore.EdgeGroup().EdgeGroup(portainer.EdgeGroupID(edgeGroupID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an Edge group with the specified identifier inside the database", err}
 	} else if err != nil {

api/http/handler/edgegroups/edgegroup_list.go

@@ -14,12 +14,12 @@ type decoratedEdgeGroup struct {
 }
 
 func (handler *Handler) edgeGroupList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	edgeGroups, err := handler.EdgeGroupService.EdgeGroups()
+	edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve Edge groups from the database", err}
 	}
 
-	edgeStacks, err := handler.EdgeStackService.EdgeStacks()
+	edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve Edge stacks from the database", err}
 	}

api/http/handler/edgegroups/edgegroup_update.go

@@ -43,7 +43,7 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	edgeGroup, err := handler.EdgeGroupService.EdgeGroup(portainer.EdgeGroupID(edgeGroupID))
+	edgeGroup, err := handler.DataStore.EdgeGroup().EdgeGroup(portainer.EdgeGroupID(edgeGroupID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an Edge group with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -51,7 +51,7 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 	}
 
 	if payload.Name != "" {
-		edgeGroups, err := handler.EdgeGroupService.EdgeGroups()
+		edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve Edge groups from the database", err}
 		}
@@ -63,12 +63,12 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 
 		edgeGroup.Name = payload.Name
 	}
-	endpoints, err := handler.EndpointService.Endpoints()
+	endpoints, err := handler.DataStore.Endpoint().Endpoints()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from database", err}
 	}
 
-	endpointGroups, err := handler.EndpointGroupService.EndpointGroups()
+	endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint groups from database", err}
 	}
@@ -81,7 +81,7 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 	} else {
 		endpointIDs := []portainer.EndpointID{}
 		for _, endpointID := range payload.Endpoints {
-			endpoint, err := handler.EndpointService.Endpoint(endpointID)
+			endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
 			if err != nil {
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint from the database", err}
 			}
@@ -97,7 +97,7 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 		edgeGroup.PartialMatch = *payload.PartialMatch
 	}
 
-	err = handler.EdgeGroupService.UpdateEdgeGroup(edgeGroup.ID, edgeGroup)
+	err = handler.DataStore.EdgeGroup().UpdateEdgeGroup(edgeGroup.ID, edgeGroup)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist Edge group changes inside the database", err}
 	}
@@ -116,27 +116,27 @@ func (handler *Handler) edgeGroupUpdate(w http.ResponseWriter, r *http.Request)
 }
 
 func (handler *Handler) updateEndpoint(endpointID portainer.EndpointID) error {
-	relation, err := handler.EndpointRelationService.EndpointRelation(endpointID)
+	relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpointID)
 	if err != nil {
 		return err
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(endpointID)
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(endpointID)
 	if err != nil {
 		return err
 	}
 
-	endpointGroup, err := handler.EndpointGroupService.EndpointGroup(endpoint.GroupID)
+	endpointGroup, err := handler.DataStore.EndpointGroup().EndpointGroup(endpoint.GroupID)
 	if err != nil {
 		return err
 	}
 
-	edgeGroups, err := handler.EdgeGroupService.EdgeGroups()
+	edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
 	if err != nil {
 		return err
 	}
 
-	edgeStacks, err := handler.EdgeStackService.EdgeStacks()
+	edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
 	if err != nil {
 		return err
 	}
@@ -150,5 +150,5 @@ func (handler *Handler) updateEndpoint(endpointID portainer.EndpointID) error {
 
 	relation.EdgeStacks = edgeStackSet
 
-	return handler.EndpointRelationService.UpdateEndpointRelation(endpoint.ID, relation)
+	return handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpoint.ID, relation)
 }

api/http/handler/edgegroups/handler.go

@@ -12,12 +12,7 @@ import (
 // Handler is the HTTP handler used to handle endpoint group operations.
 type Handler struct {
 	*mux.Router
-	EdgeGroupService        portainer.EdgeGroupService
-	EdgeStackService        portainer.EdgeStackService
-	EndpointService         portainer.EndpointService
-	EndpointGroupService    portainer.EndpointGroupService
-	EndpointRelationService portainer.EndpointRelationService
-	TagService              portainer.TagService
+	DataStore portainer.DataStore
 }
 
 // NewHandler creates a handler to manage endpoint group operations.

api/http/handler/edgestacks/edgestack_create.go

@@ -27,17 +27,17 @@ func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create Edge stack", err}
 	}
 
-	endpoints, err := handler.EndpointService.Endpoints()
+	endpoints, err := handler.DataStore.Endpoint().Endpoints()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from database", err}
 	}
 
-	endpointGroups, err := handler.EndpointGroupService.EndpointGroups()
+	endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint groups from database", err}
 	}
 
-	edgeGroups, err := handler.EdgeGroupService.EdgeGroups()
+	edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge groups from database", err}
 	}
@@ -45,14 +45,14 @@ func (handler *Handler) edgeStackCreate(w http.ResponseWriter, r *http.Request)
 	relatedEndpoints, err := portainer.EdgeStackRelatedEndpoints(edgeStack.EdgeGroups, endpoints, endpointGroups, edgeGroups)
 
 	for _, endpointID := range relatedEndpoints {
-		relation, err := handler.EndpointRelationService.EndpointRelation(endpointID)
+		relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpointID)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find endpoint relation in database", err}
 		}
 
 		relation.EdgeStacks[edgeStack.ID] = true
 
-		err = handler.EndpointRelationService.UpdateEndpointRelation(endpointID, relation)
+		err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpointID, relation)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint relation in database", err}
 		}
@@ -104,7 +104,7 @@ func (handler *Handler) createSwarmStackFromFileContent(r *http.Request) (*porta
 		return nil, err
 	}
 
-	stackID := handler.EdgeStackService.GetNextIdentifier()
+	stackID := handler.DataStore.EdgeStack().GetNextIdentifier()
 	stack := &portainer.EdgeStack{
 		ID:           portainer.EdgeStackID(stackID),
 		Name:         payload.Name,
@@ -122,7 +122,7 @@ func (handler *Handler) createSwarmStackFromFileContent(r *http.Request) (*porta
 	}
 	stack.ProjectPath = projectPath
 
-	err = handler.EdgeStackService.CreateEdgeStack(stack)
+	err = handler.DataStore.EdgeStack().CreateEdgeStack(stack)
 	if err != nil {
 		return nil, err
 	}
@@ -172,7 +172,7 @@ func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request) (*por
 		return nil, err
 	}
 
-	stackID := handler.EdgeStackService.GetNextIdentifier()
+	stackID := handler.DataStore.EdgeStack().GetNextIdentifier()
 	stack := &portainer.EdgeStack{
 		ID:           portainer.EdgeStackID(stackID),
 		Name:         payload.Name,
@@ -200,7 +200,7 @@ func (handler *Handler) createSwarmStackFromGitRepository(r *http.Request) (*por
 		return nil, err
 	}
 
-	err = handler.EdgeStackService.CreateEdgeStack(stack)
+	err = handler.DataStore.EdgeStack().CreateEdgeStack(stack)
 	if err != nil {
 		return nil, err
 	}
@@ -248,7 +248,7 @@ func (handler *Handler) createSwarmStackFromFileUpload(r *http.Request) (*portai
 		return nil, err
 	}
 
-	stackID := handler.EdgeStackService.GetNextIdentifier()
+	stackID := handler.DataStore.EdgeStack().GetNextIdentifier()
 	stack := &portainer.EdgeStack{
 		ID:           portainer.EdgeStackID(stackID),
 		Name:         payload.Name,
@@ -266,7 +266,7 @@ func (handler *Handler) createSwarmStackFromFileUpload(r *http.Request) (*portai
 	}
 	stack.ProjectPath = projectPath
 
-	err = handler.EdgeStackService.CreateEdgeStack(stack)
+	err = handler.DataStore.EdgeStack().CreateEdgeStack(stack)
 	if err != nil {
 		return nil, err
 	}
@@ -275,7 +275,7 @@ func (handler *Handler) createSwarmStackFromFileUpload(r *http.Request) (*portai
 }
 
 func (handler *Handler) validateUniqueName(name string) error {
-	edgeStacks, err := handler.EdgeStackService.EdgeStacks()
+	edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
 	if err != nil {
 		return err
 	}

api/http/handler/edgestacks/edgestack_delete.go

@@ -15,29 +15,29 @@ func (handler *Handler) edgeStackDelete(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid edge stack identifier route variable", err}
 	}
 
-	edgeStack, err := handler.EdgeStackService.EdgeStack(portainer.EdgeStackID(edgeStackID))
+	edgeStack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(edgeStackID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an edge stack with the specified identifier inside the database", err}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an edge stack with the specified identifier inside the database", err}
 	}
 
-	err = handler.EdgeStackService.DeleteEdgeStack(portainer.EdgeStackID(edgeStackID))
+	err = handler.DataStore.EdgeStack().DeleteEdgeStack(portainer.EdgeStackID(edgeStackID))
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the edge stack from the database", err}
 	}
 
-	endpoints, err := handler.EndpointService.Endpoints()
+	endpoints, err := handler.DataStore.Endpoint().Endpoints()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from database", err}
 	}
 
-	endpointGroups, err := handler.EndpointGroupService.EndpointGroups()
+	endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint groups from database", err}
 	}
 
-	edgeGroups, err := handler.EdgeGroupService.EdgeGroups()
+	edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge groups from database", err}
 	}
@@ -45,14 +45,14 @@ func (handler *Handler) edgeStackDelete(w http.ResponseWriter, r *http.Request)
 	relatedEndpoints, err := portainer.EdgeStackRelatedEndpoints(edgeStack.EdgeGroups, endpoints, endpointGroups, edgeGroups)
 
 	for _, endpointID := range relatedEndpoints {
-		relation, err := handler.EndpointRelationService.EndpointRelation(endpointID)
+		relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpointID)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find endpoint relation in database", err}
 		}
 
 		delete(relation.EdgeStacks, edgeStack.ID)
 
-		err = handler.EndpointRelationService.UpdateEndpointRelation(endpointID, relation)
+		err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpointID, relation)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint relation in database", err}
 		}

api/http/handler/edgestacks/edgestack_file.go

@@ -21,7 +21,7 @@ func (handler *Handler) edgeStackFile(w http.ResponseWriter, r *http.Request) *h
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid edge stack identifier route variable", err}
 	}
 
-	stack, err := handler.EdgeStackService.EdgeStack(portainer.EdgeStackID(stackID))
+	stack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an edge stack with the specified identifier inside the database", err}
 	} else if err != nil {

api/http/handler/edgestacks/edgestack_inspect.go

@@ -15,7 +15,7 @@ func (handler *Handler) edgeStackInspect(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid edge stack identifier route variable", err}
 	}
 
-	edgeStack, err := handler.EdgeStackService.EdgeStack(portainer.EdgeStackID(edgeStackID))
+	edgeStack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(edgeStackID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an edge stack with the specified identifier inside the database", err}
 	} else if err != nil {

api/http/handler/edgestacks/edgestack_list.go

@@ -8,7 +8,7 @@ import (
 )
 
 func (handler *Handler) edgeStackList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	edgeStacks, err := handler.EdgeStackService.EdgeStacks()
+	edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge stacks from the database", err}
 	}

api/http/handler/edgestacks/edgestack_status_update.go

@@ -35,7 +35,7 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err}
 	}
 
-	stack, err := handler.EdgeStackService.EdgeStack(portainer.EdgeStackID(stackID))
+	stack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -48,7 +48,7 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(*payload.EndpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(*payload.EndpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -66,7 +66,7 @@ func (handler *Handler) edgeStackStatusUpdate(w http.ResponseWriter, r *http.Req
 		EndpointID: *payload.EndpointID,
 	}
 
-	err = handler.EdgeStackService.UpdateEdgeStack(stack.ID, stack)
+	err = handler.DataStore.EdgeStack().UpdateEdgeStack(stack.ID, stack)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
 	}

api/http/handler/edgestacks/edgestack_update.go

@@ -34,7 +34,7 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid stack identifier route variable", err}
 	}
 
-	stack, err := handler.EdgeStackService.EdgeStack(portainer.EdgeStackID(stackID))
+	stack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(stackID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a stack with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -48,17 +48,17 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 	}
 
 	if payload.EdgeGroups != nil {
-		endpoints, err := handler.EndpointService.Endpoints()
+		endpoints, err := handler.DataStore.Endpoint().Endpoints()
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from database", err}
 		}
 
-		endpointGroups, err := handler.EndpointGroupService.EndpointGroups()
+		endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups()
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint groups from database", err}
 		}
 
-		edgeGroups, err := handler.EdgeGroupService.EdgeGroups()
+		edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge groups from database", err}
 		}
@@ -84,14 +84,14 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 		}
 
 		for endpointID := range endpointsToRemove {
-			relation, err := handler.EndpointRelationService.EndpointRelation(endpointID)
+			relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpointID)
 			if err != nil {
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find endpoint relation in database", err}
 			}
 
 			delete(relation.EdgeStacks, stack.ID)
 
-			err = handler.EndpointRelationService.UpdateEndpointRelation(endpointID, relation)
+			err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpointID, relation)
 			if err != nil {
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint relation in database", err}
 			}
@@ -105,14 +105,14 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 		}
 
 		for endpointID := range endpointsToAdd {
-			relation, err := handler.EndpointRelationService.EndpointRelation(endpointID)
+			relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpointID)
 			if err != nil {
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find endpoint relation in database", err}
 			}
 
 			relation.EdgeStacks[stack.ID] = true
 
-			err = handler.EndpointRelationService.UpdateEndpointRelation(endpointID, relation)
+			err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpointID, relation)
 			if err != nil {
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint relation in database", err}
 			}
@@ -137,7 +137,7 @@ func (handler *Handler) edgeStackUpdate(w http.ResponseWriter, r *http.Request)
 		stack.Status = map[portainer.EndpointID]portainer.EdgeStackStatus{}
 	}
 
-	err = handler.EdgeStackService.UpdateEdgeStack(stack.ID, stack)
+	err = handler.DataStore.EdgeStack().UpdateEdgeStack(stack.ID, stack)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the stack changes inside the database", err}
 	}

api/http/handler/edgestacks/handler.go

@@ -12,14 +12,10 @@ import (
 // Handler is the HTTP handler used to handle endpoint group operations.
 type Handler struct {
 	*mux.Router
-	requestBouncer          *security.RequestBouncer
-	EdgeGroupService        portainer.EdgeGroupService
-	EdgeStackService        portainer.EdgeStackService
-	EndpointService         portainer.EndpointService
-	EndpointGroupService    portainer.EndpointGroupService
-	EndpointRelationService portainer.EndpointRelationService
-	FileService             portainer.FileService
-	GitService              portainer.GitService
+	requestBouncer *security.RequestBouncer
+	DataStore      portainer.DataStore
+	FileService    portainer.FileService
+	GitService     portainer.GitService
 }
 
 // NewHandler creates a handler to manage endpoint group operations.

api/http/handler/edgetemplates/edgetemplate_list.go

@@ -2,7 +2,6 @@ package edgetemplates
 
 import (
 	"encoding/json"
-	"log"
 	"net/http"
 
 	httperror "github.com/portainer/libhttp/error"
@@ -11,35 +10,39 @@ import (
 	"github.com/portainer/portainer/api/http/client"
 )
 
+type templateFileFormat struct {
+	Version   string               `json:"version"`
+	Templates []portainer.Template `json:"templates"`
+}
+
 // GET request on /api/edgetemplates
 func (handler *Handler) edgeTemplateList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	settings, err := handler.SettingsService.Settings()
+	settings, err := handler.DataStore.Settings().Settings()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
 	}
 
-	url := portainer.EdgeTemplatesURL
+	url := portainer.DefaultTemplatesURL
 	if settings.TemplatesURL != "" {
 		url = settings.TemplatesURL
 	}
 
 	var templateData []byte
-	templateData, err = client.Get(url, 0)
+	templateData, err = client.Get(url, 10)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve external templates", err}
 	}
 
-	var templates []portainer.Template
+	var templateFile templateFileFormat
 
-	err = json.Unmarshal(templateData, &templates)
+	err = json.Unmarshal(templateData, &templateFile)
 	if err != nil {
-		log.Printf("[DEBUG] [http,edge,templates] [failed parsing edge templates] [body: %s]", templateData)
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to parse external templates", err}
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to parse template file", err}
 	}
 
-	filteredTemplates := []portainer.Template{}
+	filteredTemplates := make([]portainer.Template, 0)
 
-	for _, template := range templates {
+	for _, template := range templateFile.Templates {
 		if template.Type == portainer.EdgeStackTemplate {
 			filteredTemplates = append(filteredTemplates, template)
 		}

api/http/handler/edgetemplates/handler.go

@@ -13,8 +13,8 @@ import (
 // Handler is the HTTP handler used to handle edge endpoint operations.
 type Handler struct {
 	*mux.Router
-	requestBouncer  *security.RequestBouncer
-	SettingsService portainer.SettingsService
+	requestBouncer *security.RequestBouncer
+	DataStore      portainer.DataStore
 }
 
 // NewHandler creates a handler to manage endpoint operations.

api/http/handler/endpointedge/endpoint_edgestack_inspect.go

@@ -23,7 +23,7 @@ func (handler *Handler) endpointEdgeStackInspect(w http.ResponseWriter, r *http.
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -40,7 +40,7 @@ func (handler *Handler) endpointEdgeStackInspect(w http.ResponseWriter, r *http.
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid edge stack identifier route variable", err}
 	}
 
-	edgeStack, err := handler.EdgeStackService.EdgeStack(portainer.EdgeStackID(edgeStackID))
+	edgeStack, err := handler.DataStore.EdgeStack().EdgeStack(portainer.EdgeStackID(edgeStackID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an edge stack with the specified identifier inside the database", err}
 	} else if err != nil {

api/http/handler/endpointedge/handler.go

@@ -13,10 +13,9 @@ import (
 // Handler is the HTTP handler used to handle edge endpoint operations.
 type Handler struct {
 	*mux.Router
-	requestBouncer   *security.RequestBouncer
-	EndpointService  portainer.EndpointService
-	EdgeStackService portainer.EdgeStackService
-	FileService      portainer.FileService
+	requestBouncer *security.RequestBouncer
+	DataStore      portainer.DataStore
+	FileService    portainer.FileService
 }
 
 // NewHandler creates a handler to manage endpoint operations.

api/http/handler/endpointgroups/endpointgroup_create.go

@@ -43,12 +43,12 @@ func (handler *Handler) endpointGroupCreate(w http.ResponseWriter, r *http.Reque
 		TagIDs:             payload.TagIDs,
 	}
 
-	err = handler.EndpointGroupService.CreateEndpointGroup(endpointGroup)
+	err = handler.DataStore.EndpointGroup().CreateEndpointGroup(endpointGroup)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the endpoint group inside the database", err}
 	}
 
-	endpoints, err := handler.EndpointService.Endpoints()
+	endpoints, err := handler.DataStore.Endpoint().Endpoints()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
 	}
@@ -58,7 +58,7 @@ func (handler *Handler) endpointGroupCreate(w http.ResponseWriter, r *http.Reque
 			if endpoint.ID == id {
 				endpoint.GroupID = endpointGroup.ID
 
-				err := handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+				err := handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, &endpoint)
 				if err != nil {
 					return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update endpoint", err}
 				}
@@ -74,14 +74,14 @@ func (handler *Handler) endpointGroupCreate(w http.ResponseWriter, r *http.Reque
 	}
 
 	for _, tagID := range endpointGroup.TagIDs {
-		tag, err := handler.TagService.Tag(tagID)
+		tag, err := handler.DataStore.Tag().Tag(tagID)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve tag from the database", err}
 		}
 
 		tag.EndpointGroups[endpointGroup.ID] = true
 
-		err = handler.TagService.UpdateTag(tagID, tag)
+		err = handler.DataStore.Tag().UpdateTag(tagID, tag)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist tag changes inside the database", err}
 		}

api/http/handler/endpointgroups/endpointgroup_delete.go

@@ -20,19 +20,19 @@ func (handler *Handler) endpointGroupDelete(w http.ResponseWriter, r *http.Reque
 		return &httperror.HandlerError{http.StatusForbidden, "Unable to remove the default 'Unassigned' group", portainer.ErrCannotRemoveDefaultGroup}
 	}
 
-	endpointGroup, err := handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	endpointGroup, err := handler.DataStore.EndpointGroup().EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint group with the specified identifier inside the database", err}
 	}
 
-	err = handler.EndpointGroupService.DeleteEndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	err = handler.DataStore.EndpointGroup().DeleteEndpointGroup(portainer.EndpointGroupID(endpointGroupID))
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the endpoint group from the database", err}
 	}
 
-	endpoints, err := handler.EndpointService.Endpoints()
+	endpoints, err := handler.DataStore.Endpoint().Endpoints()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
 	}
@@ -42,7 +42,7 @@ func (handler *Handler) endpointGroupDelete(w http.ResponseWriter, r *http.Reque
 		if endpoint.GroupID == portainer.EndpointGroupID(endpointGroupID) {
 			updateAuthorizations = true
 			endpoint.GroupID = portainer.EndpointGroupID(1)
-			err = handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+			err = handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, &endpoint)
 			if err != nil {
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update endpoint", err}
 			}
@@ -62,14 +62,14 @@ func (handler *Handler) endpointGroupDelete(w http.ResponseWriter, r *http.Reque
 	}
 
 	for _, tagID := range endpointGroup.TagIDs {
-		tag, err := handler.TagService.Tag(tagID)
+		tag, err := handler.DataStore.Tag().Tag(tagID)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve tag from the database", err}
 		}
 
 		delete(tag.EndpointGroups, endpointGroup.ID)
 
-		err = handler.TagService.UpdateTag(tagID, tag)
+		err = handler.DataStore.Tag().UpdateTag(tagID, tag)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist tag changes inside the database", err}
 		}

api/http/handler/endpointgroups/endpointgroup_endpoint_add.go

@@ -21,14 +21,14 @@ func (handler *Handler) endpointGroupAddEndpoint(w http.ResponseWriter, r *http.
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
 	}
 
-	endpointGroup, err := handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	endpointGroup, err := handler.DataStore.EndpointGroup().EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint group with the specified identifier inside the database", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -37,7 +37,7 @@ func (handler *Handler) endpointGroupAddEndpoint(w http.ResponseWriter, r *http.
 
 	endpoint.GroupID = endpointGroup.ID
 
-	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
+	err = handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
 	}

api/http/handler/endpointgroups/endpointgroup_endpoint_delete.go

@@ -21,14 +21,14 @@ func (handler *Handler) endpointGroupDeleteEndpoint(w http.ResponseWriter, r *ht
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
 	}
 
-	_, err = handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	_, err = handler.DataStore.EndpointGroup().EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint group with the specified identifier inside the database", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -37,7 +37,7 @@ func (handler *Handler) endpointGroupDeleteEndpoint(w http.ResponseWriter, r *ht
 
 	endpoint.GroupID = portainer.EndpointGroupID(1)
 
-	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
+	err = handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
 	}

api/http/handler/endpointgroups/endpointgroup_inspect.go

@@ -16,7 +16,7 @@ func (handler *Handler) endpointGroupInspect(w http.ResponseWriter, r *http.Requ
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint group identifier route variable", err}
 	}
 
-	endpointGroup, err := handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	endpointGroup, err := handler.DataStore.EndpointGroup().EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
 	} else if err != nil {

api/http/handler/endpointgroups/endpointgroup_list.go

@@ -10,7 +10,7 @@ import (
 
 // GET request on /api/endpoint_groups
 func (handler *Handler) endpointGroupList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointGroups, err := handler.EndpointGroupService.EndpointGroups()
+	endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint groups from the database", err}
 	}

api/http/handler/endpointgroups/endpointgroup_update.go

@@ -35,7 +35,7 @@ func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Reque
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	endpointGroup, err := handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
+	endpointGroup, err := handler.DataStore.EndpointGroup().EndpointGroup(portainer.EndpointGroupID(endpointGroupID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint group with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -62,12 +62,12 @@ func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Reque
 			removeTags := portainer.TagDifference(endpointGroupTagSet, payloadTagSet)
 
 			for tagID := range removeTags {
-				tag, err := handler.TagService.Tag(tagID)
+				tag, err := handler.DataStore.Tag().Tag(tagID)
 				if err != nil {
 					return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a tag inside the database", err}
 				}
 				delete(tag.EndpointGroups, endpointGroup.ID)
-				err = handler.TagService.UpdateTag(tag.ID, tag)
+				err = handler.DataStore.Tag().UpdateTag(tag.ID, tag)
 				if err != nil {
 					return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist tag changes inside the database", err}
 				}
@@ -75,14 +75,14 @@ func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Reque
 
 			endpointGroup.TagIDs = payload.TagIDs
 			for _, tagID := range payload.TagIDs {
-				tag, err := handler.TagService.Tag(tagID)
+				tag, err := handler.DataStore.Tag().Tag(tagID)
 				if err != nil {
 					return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a tag inside the database", err}
 				}
 
 				tag.EndpointGroups[endpointGroup.ID] = true
 
-				err = handler.TagService.UpdateTag(tag.ID, tag)
+				err = handler.DataStore.Tag().UpdateTag(tag.ID, tag)
 				if err != nil {
 					return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist tag changes inside the database", err}
 				}
@@ -101,7 +101,7 @@ func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Reque
 		updateAuthorizations = true
 	}
 
-	err = handler.EndpointGroupService.UpdateEndpointGroup(endpointGroup.ID, endpointGroup)
+	err = handler.DataStore.EndpointGroup().UpdateEndpointGroup(endpointGroup.ID, endpointGroup)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint group changes inside the database", err}
 	}
@@ -114,7 +114,7 @@ func (handler *Handler) endpointGroupUpdate(w http.ResponseWriter, r *http.Reque
 	}
 
 	if tagsChanged {
-		endpoints, err := handler.EndpointService.Endpoints()
+		endpoints, err := handler.DataStore.Endpoint().Endpoints()
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
 

api/http/handler/endpointgroups/endpoints.go

@@ -8,7 +8,7 @@ func (handler *Handler) updateEndpointRelations(endpoint *portainer.Endpoint, en
 	}
 
 	if endpointGroup == nil {
-		unassignedGroup, err := handler.EndpointGroupService.EndpointGroup(portainer.EndpointGroupID(1))
+		unassignedGroup, err := handler.DataStore.EndpointGroup().EndpointGroup(portainer.EndpointGroupID(1))
 		if err != nil {
 			return err
 		}
@@ -16,17 +16,17 @@ func (handler *Handler) updateEndpointRelations(endpoint *portainer.Endpoint, en
 		endpointGroup = unassignedGroup
 	}
 
-	endpointRelation, err := handler.EndpointRelationService.EndpointRelation(endpoint.ID)
+	endpointRelation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpoint.ID)
 	if err != nil {
 		return err
 	}
 
-	edgeGroups, err := handler.EdgeGroupService.EdgeGroups()
+	edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
 	if err != nil {
 		return err
 	}
 
-	edgeStacks, err := handler.EdgeStackService.EdgeStacks()
+	edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
 	if err != nil {
 		return err
 	}
@@ -38,5 +38,5 @@ func (handler *Handler) updateEndpointRelations(endpoint *portainer.Endpoint, en
 	}
 	endpointRelation.EdgeStacks = stacksSet
 
-	return handler.EndpointRelationService.UpdateEndpointRelation(endpoint.ID, endpointRelation)
+	return handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpoint.ID, endpointRelation)
 }

api/http/handler/endpointgroups/handler.go

@@ -12,13 +12,8 @@ import (
 // Handler is the HTTP handler used to handle endpoint group operations.
 type Handler struct {
 	*mux.Router
-	AuthorizationService    *portainer.AuthorizationService
-	EdgeGroupService        portainer.EdgeGroupService
-	EdgeStackService        portainer.EdgeStackService
-	EndpointService         portainer.EndpointService
-	EndpointGroupService    portainer.EndpointGroupService
-	EndpointRelationService portainer.EndpointRelationService
-	TagService              portainer.TagService
+	DataStore            portainer.DataStore
+	AuthorizationService *portainer.AuthorizationService
 }
 
 // NewHandler creates a handler to manage endpoint group operations.

api/http/handler/endpointproxy/handler.go

@@ -11,9 +11,8 @@ import (
 // Handler is the HTTP handler used to proxy requests to external APIs.
 type Handler struct {
 	*mux.Router
+	DataStore            portainer.DataStore
 	requestBouncer       *security.RequestBouncer
-	EndpointService      portainer.EndpointService
-	SettingsService      portainer.SettingsService
 	ProxyManager         *proxy.Manager
 	ReverseTunnelService portainer.ReverseTunnelService
 }
@@ -24,8 +23,6 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 		Router:         mux.NewRouter(),
 		requestBouncer: bouncer,
 	}
-	h.PathPrefix("/{id}/azure").Handler(
-		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToAzureAPI)))
 	h.PathPrefix("/{id}/docker").Handler(
 		bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.proxyRequestsToDockerAPI)))
 	h.PathPrefix("/{id}/storidge").Handler(

api/http/handler/endpointproxy/proxy_azure.go

@@ -1,43 +0,0 @@
-package endpointproxy
-
-import (
-	"strconv"
-
-	httperror "github.com/portainer/libhttp/error"
-	"github.com/portainer/libhttp/request"
-	"github.com/portainer/portainer/api"
-
-	"net/http"
-)
-
-func (handler *Handler) proxyRequestsToAzureAPI(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
-	if err != nil {
-		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
-	}
-
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
-	if err == portainer.ErrObjectNotFound {
-		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
-	} else if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
-	}
-
-	err = handler.requestBouncer.AuthorizedEndpointOperation(r, endpoint, false)
-	if err != nil {
-		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access endpoint", err}
-	}
-
-	var proxy http.Handler
-	proxy = handler.ProxyManager.GetEndpointProxy(endpoint)
-	if proxy == nil {
-		proxy, err = handler.ProxyManager.CreateAndRegisterEndpointProxy(endpoint)
-		if err != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to create proxy", err}
-		}
-	}
-
-	id := strconv.Itoa(endpointID)
-	http.StripPrefix("/"+id+"/azure", proxy).ServeHTTP(w, r)
-	return nil
-}

api/http/handler/endpointproxy/proxy_docker.go

@@ -18,7 +18,7 @@ func (handler *Handler) proxyRequestsToDockerAPI(w http.ResponseWriter, r *http.
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -44,7 +44,7 @@ func (handler *Handler) proxyRequestsToDockerAPI(w http.ResponseWriter, r *http.
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update tunnel status", err}
 			}
 
-			settings, err := handler.SettingsService.Settings()
+			settings, err := handler.DataStore.Settings().Settings()
 			if err != nil {
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
 			}

api/http/handler/endpointproxy/proxy_storidge.go

@@ -18,7 +18,7 @@ func (handler *Handler) proxyRequestsToStoridgeAPI(w http.ResponseWriter, r *htt
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {

api/http/handler/endpoints/endpoint_create.go

@@ -18,21 +18,18 @@ import (
 )
 
 type endpointCreatePayload struct {
-	Name                   string
-	URL                    string
-	EndpointType           int
-	PublicURL              string
-	GroupID                int
-	TLS                    bool
-	TLSSkipVerify          bool
-	TLSSkipClientVerify    bool
-	TLSCACertFile          []byte
-	TLSCertFile            []byte
-	TLSKeyFile             []byte
-	AzureApplicationID     string
-	AzureTenantID          string
-	AzureAuthenticationKey string
-	TagIDs                 []portainer.TagID
+	Name                string
+	URL                 string
+	EndpointType        int
+	PublicURL           string
+	GroupID             int
+	TLS                 bool
+	TLSSkipVerify       bool
+	TLSSkipClientVerify bool
+	TLSCACertFile       []byte
+	TLSCertFile         []byte
+	TLSKeyFile          []byte
+	TagIDs              []portainer.TagID
 }
 
 func (payload *endpointCreatePayload) Validate(r *http.Request) error {
@@ -44,7 +41,7 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 
 	endpointType, err := request.RetrieveNumericMultiPartFormValue(r, "EndpointType", false)
 	if err != nil || endpointType == 0 {
-		return portainer.Error("Invalid endpoint type value. Value must be one of: 1 (Docker environment), 2 (Agent environment), 3 (Azure environment) or 4 (Edge Agent environment)")
+		return portainer.Error("Invalid endpoint type value. Value must be one of: 1 (Docker environment), 2 (Agent environment) or 4 (Edge Agent environment)")
 	}
 	payload.EndpointType = endpointType
 
@@ -96,45 +93,20 @@ func (payload *endpointCreatePayload) Validate(r *http.Request) error {
 		}
 	}
 
-	switch portainer.EndpointType(payload.EndpointType) {
-	case portainer.AzureEnvironment:
-		azureApplicationID, err := request.RetrieveMultiPartFormValue(r, "AzureApplicationID", false)
-		if err != nil {
-			return portainer.Error("Invalid Azure application ID")
-		}
-		payload.AzureApplicationID = azureApplicationID
-
-		azureTenantID, err := request.RetrieveMultiPartFormValue(r, "AzureTenantID", false)
-		if err != nil {
-			return portainer.Error("Invalid Azure tenant ID")
-		}
-		payload.AzureTenantID = azureTenantID
-
-		azureAuthenticationKey, err := request.RetrieveMultiPartFormValue(r, "AzureAuthenticationKey", false)
-		if err != nil {
-			return portainer.Error("Invalid Azure authentication key")
-		}
-		payload.AzureAuthenticationKey = azureAuthenticationKey
-	default:
-		url, err := request.RetrieveMultiPartFormValue(r, "URL", true)
-		if err != nil {
-			return portainer.Error("Invalid endpoint URL")
-		}
-		payload.URL = url
-
-		publicURL, _ := request.RetrieveMultiPartFormValue(r, "PublicURL", true)
-		payload.PublicURL = publicURL
+	endpointURL, err := request.RetrieveMultiPartFormValue(r, "URL", true)
+	if err != nil {
+		return portainer.Error("Invalid endpoint URL")
 	}
+	payload.URL = endpointURL
+
+	publicURL, _ := request.RetrieveMultiPartFormValue(r, "PublicURL", true)
+	payload.PublicURL = publicURL
 
 	return nil
 }
 
 // POST request on /api/endpoints
 func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	if !handler.authorizeEndpointManagement {
-		return &httperror.HandlerError{http.StatusServiceUnavailable, "Endpoint management is disabled", ErrEndpointManagementDisabled}
-	}
-
 	payload := &endpointCreatePayload{}
 	err := payload.Validate(r)
 	if err != nil {
@@ -146,17 +118,17 @@ func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *
 		return endpointCreationError
 	}
 
-	endpointGroup, err := handler.EndpointGroupService.EndpointGroup(endpoint.GroupID)
+	endpointGroup, err := handler.DataStore.EndpointGroup().EndpointGroup(endpoint.GroupID)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint group inside the database", err}
 	}
 
-	edgeGroups, err := handler.EdgeGroupService.EdgeGroups()
+	edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge groups from the database", err}
 	}
 
-	edgeStacks, err := handler.EdgeStackService.EdgeStacks()
+	edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge stacks from the database", err}
 	}
@@ -173,7 +145,7 @@ func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
-	err = handler.EndpointRelationService.CreateEndpointRelation(relationObject)
+	err = handler.DataStore.EndpointRelation().CreateEndpointRelation(relationObject)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the relation object inside the database", err}
 	}
@@ -182,9 +154,7 @@ func (handler *Handler) endpointCreate(w http.ResponseWriter, r *http.Request) *
 }
 
 func (handler *Handler) createEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
-	if portainer.EndpointType(payload.EndpointType) == portainer.AzureEnvironment {
-		return handler.createAzureEndpoint(payload)
-	} else if portainer.EndpointType(payload.EndpointType) == portainer.EdgeAgentEnvironment {
+	if portainer.EndpointType(payload.EndpointType) == portainer.EdgeAgentEnvironment {
 		return handler.createEdgeAgentEndpoint(payload)
 	}
 
@@ -194,47 +164,9 @@ func (handler *Handler) createEndpoint(payload *endpointCreatePayload) (*portain
 	return handler.createUnsecuredEndpoint(payload)
 }
 
-func (handler *Handler) createAzureEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
-	credentials := portainer.AzureCredentials{
-		ApplicationID:     payload.AzureApplicationID,
-		TenantID:          payload.AzureTenantID,
-		AuthenticationKey: payload.AzureAuthenticationKey,
-	}
-
-	httpClient := client.NewHTTPClient()
-	_, err := httpClient.ExecuteAzureAuthenticationRequest(&credentials)
-	if err != nil {
-		return nil, &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate against Azure", err}
-	}
-
-	endpointID := handler.EndpointService.GetNextIdentifier()
-	endpoint := &portainer.Endpoint{
-		ID:                 portainer.EndpointID(endpointID),
-		Name:               payload.Name,
-		URL:                "https://management.azure.com",
-		Type:               portainer.AzureEnvironment,
-		GroupID:            portainer.EndpointGroupID(payload.GroupID),
-		PublicURL:          payload.PublicURL,
-		UserAccessPolicies: portainer.UserAccessPolicies{},
-		TeamAccessPolicies: portainer.TeamAccessPolicies{},
-		Extensions:         []portainer.EndpointExtension{},
-		AzureCredentials:   credentials,
-		TagIDs:             payload.TagIDs,
-		Status:             portainer.EndpointStatusUp,
-		Snapshots:          []portainer.Snapshot{},
-	}
-
-	err = handler.saveEndpointAndUpdateAuthorizations(endpoint)
-	if err != nil {
-		return nil, &httperror.HandlerError{http.StatusInternalServerError, "An error occured while trying to create the endpoint", err}
-	}
-
-	return endpoint, nil
-}
-
 func (handler *Handler) createEdgeAgentEndpoint(payload *endpointCreatePayload) (*portainer.Endpoint, *httperror.HandlerError) {
 	endpointType := portainer.EdgeAgentEnvironment
-	endpointID := handler.EndpointService.GetNextIdentifier()
+	endpointID := handler.DataStore.Endpoint().GetNextIdentifier()
 
 	portainerURL, err := url.Parse(payload.URL)
 	if err != nil {
@@ -296,7 +228,7 @@ func (handler *Handler) createUnsecuredEndpoint(payload *endpointCreatePayload)
 		}
 	}
 
-	endpointID := handler.EndpointService.GetNextIdentifier()
+	endpointID := handler.DataStore.Endpoint().GetNextIdentifier()
 	endpoint := &portainer.Endpoint{
 		ID:        portainer.EndpointID(endpointID),
 		Name:      payload.Name,
@@ -339,7 +271,7 @@ func (handler *Handler) createTLSSecuredEndpoint(payload *endpointCreatePayload)
 		endpointType = portainer.AgentOnDockerEnvironment
 	}
 
-	endpointID := handler.EndpointService.GetNextIdentifier()
+	endpointID := handler.DataStore.Endpoint().GetNextIdentifier()
 	endpoint := &portainer.Endpoint{
 		ID:        portainer.EndpointID(endpointID),
 		Name:      payload.Name,
@@ -395,12 +327,12 @@ func (handler *Handler) snapshotAndPersistEndpoint(endpoint *portainer.Endpoint)
 }
 
 func (handler *Handler) saveEndpointAndUpdateAuthorizations(endpoint *portainer.Endpoint) error {
-	err := handler.EndpointService.CreateEndpoint(endpoint)
+	err := handler.DataStore.Endpoint().CreateEndpoint(endpoint)
 	if err != nil {
 		return err
 	}
 
-	group, err := handler.EndpointGroupService.EndpointGroup(endpoint.GroupID)
+	group, err := handler.DataStore.EndpointGroup().EndpointGroup(endpoint.GroupID)
 	if err != nil {
 		return err
 	}
@@ -410,14 +342,14 @@ func (handler *Handler) saveEndpointAndUpdateAuthorizations(endpoint *portainer.
 	}
 
 	for _, tagID := range endpoint.TagIDs {
-		tag, err := handler.TagService.Tag(tagID)
+		tag, err := handler.DataStore.Tag().Tag(tagID)
 		if err != nil {
 			return err
 		}
 
 		tag.Endpoints[endpoint.ID] = true
 
-		err = handler.TagService.UpdateTag(tagID, tag)
+		err = handler.DataStore.Tag().UpdateTag(tagID, tag)
 		if err != nil {
 			return err
 		}

api/http/handler/endpoints/endpoint_delete.go

@@ -12,16 +12,12 @@ import (
 
 // DELETE request on /api/endpoints/:id
 func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	if !handler.authorizeEndpointManagement {
-		return &httperror.HandlerError{http.StatusServiceUnavailable, "Endpoint management is disabled", ErrEndpointManagementDisabled}
-	}
-
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -36,7 +32,7 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
-	err = handler.EndpointService.DeleteEndpoint(portainer.EndpointID(endpointID))
+	err = handler.DataStore.Endpoint().DeleteEndpoint(portainer.EndpointID(endpointID))
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove endpoint from the database", err}
 	}
@@ -50,26 +46,26 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
-	err = handler.EndpointRelationService.DeleteEndpointRelation(endpoint.ID)
+	err = handler.DataStore.EndpointRelation().DeleteEndpointRelation(endpoint.ID)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove endpoint relation from the database", err}
 	}
 
 	for _, tagID := range endpoint.TagIDs {
-		tag, err := handler.TagService.Tag(tagID)
+		tag, err := handler.DataStore.Tag().Tag(tagID)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusNotFound, "Unable to find tag inside the database", err}
 		}
 
 		delete(tag.Endpoints, endpoint.ID)
 
-		err = handler.TagService.UpdateTag(tagID, tag)
+		err = handler.DataStore.Tag().UpdateTag(tagID, tag)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist tag relation inside the database", err}
 		}
 	}
 
-	edgeGroups, err := handler.EdgeGroupService.EdgeGroups()
+	edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge groups from the database", err}
 	}
@@ -79,14 +75,14 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 		endpointIdx := findEndpointIndex(edgeGroup.Endpoints, endpoint.ID)
 		if endpointIdx != -1 {
 			edgeGroup.Endpoints = removeElement(edgeGroup.Endpoints, endpointIdx)
-			err = handler.EdgeGroupService.UpdateEdgeGroup(edgeGroup.ID, edgeGroup)
+			err = handler.DataStore.EdgeGroup().UpdateEdgeGroup(edgeGroup.ID, edgeGroup)
 			if err != nil {
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update edge group", err}
 			}
 		}
 	}
 
-	edgeStacks, err := handler.EdgeStackService.EdgeStacks()
+	edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge stacks from the database", err}
 	}
@@ -95,7 +91,7 @@ func (handler *Handler) endpointDelete(w http.ResponseWriter, r *http.Request) *
 		edgeStack := &edgeStacks[idx]
 		if _, ok := edgeStack.Status[endpoint.ID]; ok {
 			delete(edgeStack.Status, endpoint.ID)
-			err = handler.EdgeStackService.UpdateEdgeStack(edgeStack.ID, edgeStack)
+			err = handler.DataStore.EdgeStack().UpdateEdgeStack(edgeStack.ID, edgeStack)
 			if err != nil {
 				return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update edge stack", err}
 			}

api/http/handler/endpoints/endpoint_extension_add.go

@@ -34,7 +34,7 @@ func (handler *Handler) endpointExtensionAdd(w http.ResponseWriter, r *http.Requ
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -66,7 +66,7 @@ func (handler *Handler) endpointExtensionAdd(w http.ResponseWriter, r *http.Requ
 		endpoint.Extensions = append(endpoint.Extensions, *extension)
 	}
 
-	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
+	err = handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
 	}

api/http/handler/endpoints/endpoint_extension_remove.go

@@ -18,7 +18,7 @@ func (handler *Handler) endpointExtensionRemove(w http.ResponseWriter, r *http.R
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -36,7 +36,7 @@ func (handler *Handler) endpointExtensionRemove(w http.ResponseWriter, r *http.R
 		}
 	}
 
-	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
+	err = handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
 	}

api/http/handler/endpoints/endpoint_inspect.go

@@ -16,7 +16,7 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {

api/http/handler/endpoints/endpoint_job.go

@@ -63,7 +63,7 @@ func (handler *Handler) endpointJob(w http.ResponseWriter, r *http.Request) *htt
 
 	nodeName, _ := request.RetrieveQueryParameter(r, "nodeName", true)
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {

api/http/handler/endpoints/endpoint_list.go

@@ -38,12 +38,12 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 	var endpointIDs []portainer.EndpointID
 	request.RetrieveJSONQueryParameter(r, "endpointIds", &endpointIDs, true)
 
-	endpointGroups, err := handler.EndpointGroupService.EndpointGroups()
+	endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoint groups from the database", err}
 	}
 
-	endpoints, err := handler.EndpointService.Endpoints()
+	endpoints, err := handler.DataStore.Endpoint().Endpoints()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
 	}
@@ -64,7 +64,7 @@ func (handler *Handler) endpointList(w http.ResponseWriter, r *http.Request) *ht
 	}
 
 	if search != "" {
-		tags, err := handler.TagService.Tags()
+		tags, err := handler.DataStore.Tag().Tags()
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve tags from the database", err}
 		}

api/http/handler/endpoints/endpoint_snapshot.go

@@ -16,20 +16,16 @@ func (handler *Handler) endpointSnapshot(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find an endpoint with the specified identifier inside the database", err}
 	}
 
-	if endpoint.Type == portainer.AzureEnvironment {
-		return &httperror.HandlerError{http.StatusBadRequest, "Snapshots not supported for Azure endpoints", err}
-	}
-
 	snapshot, snapshotError := handler.Snapshotter.CreateSnapshot(endpoint)
 
-	latestEndpointReference, err := handler.EndpointService.Endpoint(endpoint.ID)
+	latestEndpointReference, err := handler.DataStore.Endpoint().Endpoint(endpoint.ID)
 	if latestEndpointReference == nil {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	}
@@ -43,7 +39,7 @@ func (handler *Handler) endpointSnapshot(w http.ResponseWriter, r *http.Request)
 		latestEndpointReference.Snapshots = []portainer.Snapshot{*snapshot}
 	}
 
-	err = handler.EndpointService.UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
+	err = handler.DataStore.Endpoint().UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
 	}

api/http/handler/endpoints/endpoint_snapshots.go

@@ -11,19 +11,15 @@ import (
 
 // POST request on /api/endpoints/snapshot
 func (handler *Handler) endpointSnapshots(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	endpoints, err := handler.EndpointService.Endpoints()
+	endpoints, err := handler.DataStore.Endpoint().Endpoints()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve endpoints from the database", err}
 	}
 
 	for _, endpoint := range endpoints {
-		if endpoint.Type == portainer.AzureEnvironment {
-			continue
-		}
-
 		snapshot, snapshotError := handler.Snapshotter.CreateSnapshot(&endpoint)
 
-		latestEndpointReference, err := handler.EndpointService.Endpoint(endpoint.ID)
+		latestEndpointReference, err := handler.DataStore.Endpoint().Endpoint(endpoint.ID)
 		if latestEndpointReference == nil {
 			log.Printf("background schedule error (endpoint snapshot). Endpoint not found inside the database anymore (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
 			continue
@@ -39,7 +35,7 @@ func (handler *Handler) endpointSnapshots(w http.ResponseWriter, r *http.Request
 			latestEndpointReference.Snapshots = []portainer.Snapshot{*snapshot}
 		}
 
-		err = handler.EndpointService.UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
+		err = handler.DataStore.Endpoint().UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
 		}

api/http/handler/endpoints/endpoint_status_inspect.go

@@ -30,7 +30,7 @@ func (handler *Handler) endpointStatusInspect(w http.ResponseWriter, r *http.Req
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -47,13 +47,13 @@ func (handler *Handler) endpointStatusInspect(w http.ResponseWriter, r *http.Req
 
 		endpoint.EdgeID = edgeIdentifier
 
-		err := handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
+		err := handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to Unable to persist endpoint changes inside the database", err}
 		}
 	}
 
-	settings, err := handler.SettingsService.Settings()
+	settings, err := handler.DataStore.Settings().Settings()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
 	}
@@ -72,14 +72,14 @@ func (handler *Handler) endpointStatusInspect(w http.ResponseWriter, r *http.Req
 		handler.ReverseTunnelService.SetTunnelStatusToActive(endpoint.ID)
 	}
 
-	relation, err := handler.EndpointRelationService.EndpointRelation(endpoint.ID)
+	relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpoint.ID)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve relation object from the database", err}
 	}
 
 	edgeStacksStatus := []stackStatusResponse{}
 	for stackID := range relation.EdgeStacks {
-		stack, err := handler.EdgeStackService.EdgeStack(stackID)
+		stack, err := handler.DataStore.EdgeStack().EdgeStack(stackID)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge stack from the database", err}
 		}

api/http/handler/endpoints/endpoint_update.go

@@ -9,24 +9,20 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	"github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/http/client"
 )
 
 type endpointUpdatePayload struct {
-	Name                   *string
-	URL                    *string
-	PublicURL              *string
-	GroupID                *int
-	TLS                    *bool
-	TLSSkipVerify          *bool
-	TLSSkipClientVerify    *bool
-	Status                 *int
-	AzureApplicationID     *string
-	AzureTenantID          *string
-	AzureAuthenticationKey *string
-	TagIDs                 []portainer.TagID
-	UserAccessPolicies     portainer.UserAccessPolicies
-	TeamAccessPolicies     portainer.TeamAccessPolicies
+	Name                *string
+	URL                 *string
+	PublicURL           *string
+	GroupID             *int
+	TLS                 *bool
+	TLSSkipVerify       *bool
+	TLSSkipClientVerify *bool
+	Status              *int
+	TagIDs              []portainer.TagID
+	UserAccessPolicies  portainer.UserAccessPolicies
+	TeamAccessPolicies  portainer.TeamAccessPolicies
 }
 
 func (payload *endpointUpdatePayload) Validate(r *http.Request) error {
@@ -35,10 +31,6 @@ func (payload *endpointUpdatePayload) Validate(r *http.Request) error {
 
 // PUT request on /api/endpoints/:id
 func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	if !handler.authorizeEndpointManagement {
-		return &httperror.HandlerError{http.StatusServiceUnavailable, "Endpoint management is disabled", ErrEndpointManagementDisabled}
-	}
-
 	endpointID, err := request.RetrieveNumericRouteVariableValue(r, "id")
 	if err != nil {
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid endpoint identifier route variable", err}
@@ -50,7 +42,7 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	endpoint, err := handler.EndpointService.Endpoint(portainer.EndpointID(endpointID))
+	endpoint, err := handler.DataStore.Endpoint().Endpoint(portainer.EndpointID(endpointID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find an endpoint with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -88,13 +80,13 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 			removeTags := portainer.TagDifference(endpointTagSet, payloadTagSet)
 
 			for tagID := range removeTags {
-				tag, err := handler.TagService.Tag(tagID)
+				tag, err := handler.DataStore.Tag().Tag(tagID)
 				if err != nil {
 					return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a tag inside the database", err}
 				}
 
 				delete(tag.Endpoints, endpoint.ID)
-				err = handler.TagService.UpdateTag(tag.ID, tag)
+				err = handler.DataStore.Tag().UpdateTag(tag.ID, tag)
 				if err != nil {
 					return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist tag changes inside the database", err}
 				}
@@ -102,14 +94,14 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 
 			endpoint.TagIDs = payload.TagIDs
 			for _, tagID := range payload.TagIDs {
-				tag, err := handler.TagService.Tag(tagID)
+				tag, err := handler.DataStore.Tag().Tag(tagID)
 				if err != nil {
 					return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a tag inside the database", err}
 				}
 
 				tag.Endpoints[endpoint.ID] = true
 
-				err = handler.TagService.UpdateTag(tag.ID, tag)
+				err = handler.DataStore.Tag().UpdateTag(tag.ID, tag)
 				if err != nil {
 					return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist tag changes inside the database", err}
 				}
@@ -141,26 +133,6 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
-	if endpoint.Type == portainer.AzureEnvironment {
-		credentials := endpoint.AzureCredentials
-		if payload.AzureApplicationID != nil {
-			credentials.ApplicationID = *payload.AzureApplicationID
-		}
-		if payload.AzureTenantID != nil {
-			credentials.TenantID = *payload.AzureTenantID
-		}
-		if payload.AzureAuthenticationKey != nil {
-			credentials.AuthenticationKey = *payload.AzureAuthenticationKey
-		}
-
-		httpClient := client.NewHTTPClient()
-		_, authErr := httpClient.ExecuteAzureAuthenticationRequest(&credentials)
-		if authErr != nil {
-			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to authenticate against Azure", authErr}
-		}
-		endpoint.AzureCredentials = credentials
-	}
-
 	if payload.TLS != nil {
 		folder := strconv.Itoa(endpointID)
 
@@ -205,14 +177,14 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 		}
 	}
 
-	if payload.URL != nil || payload.TLS != nil || endpoint.Type == portainer.AzureEnvironment {
+	if payload.URL != nil || payload.TLS != nil {
 		_, err = handler.ProxyManager.CreateAndRegisterEndpointProxy(endpoint)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to register HTTP proxy for the endpoint", err}
 		}
 	}
 
-	err = handler.EndpointService.UpdateEndpoint(endpoint.ID, endpoint)
+	err = handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
 	}
@@ -225,22 +197,22 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 	}
 
 	if endpoint.Type == portainer.EdgeAgentEnvironment && (groupIDChanged || tagsChanged) {
-		relation, err := handler.EndpointRelationService.EndpointRelation(endpoint.ID)
+		relation, err := handler.DataStore.EndpointRelation().EndpointRelation(endpoint.ID)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find endpoint relation inside the database", err}
 		}
 
-		endpointGroup, err := handler.EndpointGroupService.EndpointGroup(endpoint.GroupID)
+		endpointGroup, err := handler.DataStore.EndpointGroup().EndpointGroup(endpoint.GroupID)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find endpoint group inside the database", err}
 		}
 
-		edgeGroups, err := handler.EdgeGroupService.EdgeGroups()
+		edgeGroups, err := handler.DataStore.EdgeGroup().EdgeGroups()
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge groups from the database", err}
 		}
 
-		edgeStacks, err := handler.EdgeStackService.EdgeStacks()
+		edgeStacks, err := handler.DataStore.EdgeStack().EdgeStacks()
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve edge stacks from the database", err}
 		}
@@ -254,7 +226,7 @@ func (handler *Handler) endpointUpdate(w http.ResponseWriter, r *http.Request) *
 
 		relation.EdgeStacks = edgeStackSet
 
-		err = handler.EndpointRelationService.UpdateEndpointRelation(endpoint.ID, relation)
+		err = handler.DataStore.EndpointRelation().UpdateEndpointRelation(endpoint.ID, relation)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint relation changes inside the database", err}
 		}

api/http/handler/endpoints/handler.go

@@ -11,14 +11,7 @@ import (
 	"github.com/gorilla/mux"
 )
 
-const (
-	// ErrEndpointManagementDisabled is an error raised when trying to access the endpoints management endpoints
-	// when the server has been started with the --external-endpoints flag
-	ErrEndpointManagementDisabled = portainer.Error("Endpoint management is disabled")
-)
-
 func hideFields(endpoint *portainer.Endpoint) {
-	endpoint.AzureCredentials = portainer.AzureCredentials{}
 	if len(endpoint.Snapshots) > 0 {
 		endpoint.Snapshots[0].SnapshotRaw = portainer.SnapshotRaw{}
 	}
@@ -27,29 +20,21 @@ func hideFields(endpoint *portainer.Endpoint) {
 // Handler is the HTTP handler used to handle endpoint operations.
 type Handler struct {
 	*mux.Router
-	authorizeEndpointManagement bool
-	requestBouncer              *security.RequestBouncer
-	AuthorizationService        *portainer.AuthorizationService
-	EdgeGroupService            portainer.EdgeGroupService
-	EdgeStackService            portainer.EdgeStackService
-	EndpointService             portainer.EndpointService
-	EndpointGroupService        portainer.EndpointGroupService
-	EndpointRelationService     portainer.EndpointRelationService
-	FileService                 portainer.FileService
-	JobService                  portainer.JobService
-	ProxyManager                *proxy.Manager
-	ReverseTunnelService        portainer.ReverseTunnelService
-	SettingsService             portainer.SettingsService
-	Snapshotter                 portainer.Snapshotter
-	TagService                  portainer.TagService
+	requestBouncer       *security.RequestBouncer
+	DataStore            portainer.DataStore
+	AuthorizationService *portainer.AuthorizationService
+	FileService          portainer.FileService
+	JobService           portainer.JobService
+	ProxyManager         *proxy.Manager
+	ReverseTunnelService portainer.ReverseTunnelService
+	Snapshotter          portainer.Snapshotter
 }
 
 // NewHandler creates a handler to manage endpoint operations.
-func NewHandler(bouncer *security.RequestBouncer, authorizeEndpointManagement bool) *Handler {
+func NewHandler(bouncer *security.RequestBouncer) *Handler {
 	h := &Handler{
-		Router:                      mux.NewRouter(),
-		authorizeEndpointManagement: authorizeEndpointManagement,
-		requestBouncer:              bouncer,
+		Router:         mux.NewRouter(),
+		requestBouncer: bouncer,
 	}
 
 	h.Handle("/endpoints",

api/http/handler/extensions/data.go

@@ -17,7 +17,7 @@ func updateTeamAccessPolicyToReadOnlyRole(policies portainer.TeamAccessPolicies,
 }
 
 func (handler *Handler) upgradeRBACData() error {
-	endpointGroups, err := handler.EndpointGroupService.EndpointGroups()
+	endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups()
 	if err != nil {
 		return err
 	}
@@ -31,13 +31,13 @@ func (handler *Handler) upgradeRBACData() error {
 			updateTeamAccessPolicyToReadOnlyRole(endpointGroup.TeamAccessPolicies, key)
 		}
 
-		err := handler.EndpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+		err := handler.DataStore.EndpointGroup().UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
 		if err != nil {
 			return err
 		}
 	}
 
-	endpoints, err := handler.EndpointService.Endpoints()
+	endpoints, err := handler.DataStore.Endpoint().Endpoints()
 	if err != nil {
 		return err
 	}
@@ -51,7 +51,7 @@ func (handler *Handler) upgradeRBACData() error {
 			updateTeamAccessPolicyToReadOnlyRole(endpoint.TeamAccessPolicies, key)
 		}
 
-		err := handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		err := handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, &endpoint)
 		if err != nil {
 			return err
 		}
@@ -73,7 +73,7 @@ func updateTeamAccessPolicyToNoRole(policies portainer.TeamAccessPolicies, key p
 }
 
 func (handler *Handler) downgradeRBACData() error {
-	endpointGroups, err := handler.EndpointGroupService.EndpointGroups()
+	endpointGroups, err := handler.DataStore.EndpointGroup().EndpointGroups()
 	if err != nil {
 		return err
 	}
@@ -87,13 +87,13 @@ func (handler *Handler) downgradeRBACData() error {
 			updateTeamAccessPolicyToNoRole(endpointGroup.TeamAccessPolicies, key)
 		}
 
-		err := handler.EndpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+		err := handler.DataStore.EndpointGroup().UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
 		if err != nil {
 			return err
 		}
 	}
 
-	endpoints, err := handler.EndpointService.Endpoints()
+	endpoints, err := handler.DataStore.Endpoint().Endpoints()
 	if err != nil {
 		return err
 	}
@@ -107,7 +107,7 @@ func (handler *Handler) downgradeRBACData() error {
 			updateTeamAccessPolicyToNoRole(endpoint.TeamAccessPolicies, key)
 		}
 
-		err := handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		err := handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, &endpoint)
 		if err != nil {
 			return err
 		}

api/http/handler/extensions/extension_create.go

@@ -36,7 +36,7 @@ func (handler *Handler) extensionCreate(w http.ResponseWriter, r *http.Request)
 	}
 	extensionID := portainer.ExtensionID(extensionIdentifier)
 
-	extensions, err := handler.ExtensionService.Extensions()
+	extensions, err := handler.DataStore.Extension().Extensions()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extensions status from the database", err}
 	}
@@ -77,7 +77,7 @@ func (handler *Handler) extensionCreate(w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	err = handler.ExtensionService.Persist(extension)
+	err = handler.DataStore.Extension().Persist(extension)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist extension status inside the database", err}
 	}

api/http/handler/extensions/extension_delete.go

@@ -17,7 +17,7 @@ func (handler *Handler) extensionDelete(w http.ResponseWriter, r *http.Request)
 	}
 	extensionID := portainer.ExtensionID(extensionIdentifier)
 
-	extension, err := handler.ExtensionService.Extension(extensionID)
+	extension, err := handler.DataStore.Extension().Extension(extensionID)
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a extension with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -36,7 +36,7 @@ func (handler *Handler) extensionDelete(w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	err = handler.ExtensionService.DeleteExtension(extensionID)
+	err = handler.DataStore.Extension().DeleteExtension(extensionID)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to delete the extension from the database", err}
 	}

api/http/handler/extensions/extension_inspect.go

@@ -25,7 +25,7 @@ func (handler *Handler) extensionInspect(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extensions informations", err}
 	}
 
-	localExtension, err := handler.ExtensionService.Extension(extensionID)
+	localExtension, err := handler.DataStore.Extension().Extension(extensionID)
 	if err != nil && err != portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extension information from the database", err}
 	}

api/http/handler/extensions/extension_list.go

@@ -12,7 +12,7 @@ import (
 func (handler *Handler) extensionList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
 	fetchManifestInformation, _ := request.RetrieveBooleanQueryParameter(r, "store", true)
 
-	extensions, err := handler.ExtensionService.Extensions()
+	extensions, err := handler.DataStore.Extension().Extensions()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve extensions from the database", err}
 	}

api/http/handler/extensions/extension_update.go

@@ -35,7 +35,7 @@ func (handler *Handler) extensionUpdate(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	extension, err := handler.ExtensionService.Extension(extensionID)
+	extension, err := handler.DataStore.Extension().Extension(extensionID)
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a extension with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -47,7 +47,7 @@ func (handler *Handler) extensionUpdate(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to update extension", err}
 	}
 
-	err = handler.ExtensionService.Persist(extension)
+	err = handler.DataStore.Extension().Persist(extension)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist extension status inside the database", err}
 	}

api/http/handler/extensions/extension_upload.go

@@ -66,7 +66,7 @@ func (handler *Handler) extensionUpload(w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	err = handler.ExtensionService.Persist(extension)
+	err = handler.DataStore.Extension().Persist(extension)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist extension status inside the database", err}
 	}

api/http/handler/extensions/handler.go

@@ -14,11 +14,8 @@ import (
 // Handler is the HTTP handler used to handle extension operations.
 type Handler struct {
 	*mux.Router
-	ExtensionService     portainer.ExtensionService
+	DataStore            portainer.DataStore
 	ExtensionManager     portainer.ExtensionManager
-	EndpointGroupService portainer.EndpointGroupService
-	EndpointService      portainer.EndpointService
-	RegistryService      portainer.RegistryService
 	AuthorizationService *portainer.AuthorizationService
 }
 

api/http/handler/handler.go

@@ -90,8 +90,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			http.StripPrefix("/api/endpoints", h.EndpointProxyHandler).ServeHTTP(w, r)
 		case strings.Contains(r.URL.Path, "/storidge/"):
 			http.StripPrefix("/api/endpoints", h.EndpointProxyHandler).ServeHTTP(w, r)
-		case strings.Contains(r.URL.Path, "/azure/"):
-			http.StripPrefix("/api/endpoints", h.EndpointProxyHandler).ServeHTTP(w, r)
 		case strings.Contains(r.URL.Path, "/edge/"):
 			http.StripPrefix("/api/endpoints", h.EndpointEdgeHandler).ServeHTTP(w, r)
 		default:

api/http/handler/registries/handler.go

@@ -18,11 +18,10 @@ func hideFields(registry *portainer.Registry) {
 // Handler is the HTTP handler used to handle registry operations.
 type Handler struct {
 	*mux.Router
-	requestBouncer   *security.RequestBouncer
-	RegistryService  portainer.RegistryService
-	ExtensionService portainer.ExtensionService
-	FileService      portainer.FileService
-	ProxyManager     *proxy.Manager
+	requestBouncer *security.RequestBouncer
+	DataStore      portainer.DataStore
+	FileService    portainer.FileService
+	ProxyManager   *proxy.Manager
 }
 
 // NewHandler creates a handler to manage registry operations.

api/http/handler/registries/proxy.go

@@ -17,7 +17,7 @@ func (handler *Handler) proxyRequestsToRegistryAPI(w http.ResponseWriter, r *htt
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry identifier route variable", err}
 	}
 
-	registry, err := handler.RegistryService.Registry(portainer.RegistryID(registryID))
+	registry, err := handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a registry with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -29,7 +29,7 @@ func (handler *Handler) proxyRequestsToRegistryAPI(w http.ResponseWriter, r *htt
 		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access registry", portainer.ErrEndpointAccessDenied}
 	}
 
-	extension, err := handler.ExtensionService.Extension(portainer.RegistryManagementExtension)
+	extension, err := handler.DataStore.Extension().Extension(portainer.RegistryManagementExtension)
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Registry management extension is not enabled", err}
 	} else if err != nil {

api/http/handler/registries/proxy_management_gitlab.go

@@ -17,7 +17,7 @@ func (handler *Handler) proxyRequestsToGitlabAPIWithRegistry(w http.ResponseWrit
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry identifier route variable", err}
 	}
 
-	registry, err := handler.RegistryService.Registry(portainer.RegistryID(registryID))
+	registry, err := handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a registry with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -29,7 +29,7 @@ func (handler *Handler) proxyRequestsToGitlabAPIWithRegistry(w http.ResponseWrit
 		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to access registry", portainer.ErrEndpointAccessDenied}
 	}
 
-	extension, err := handler.ExtensionService.Extension(portainer.RegistryManagementExtension)
+	extension, err := handler.DataStore.Extension().Extension(portainer.RegistryManagementExtension)
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Registry management extension is not enabled", err}
 	} else if err != nil {

api/http/handler/registries/registry_configure.go

@@ -78,7 +78,7 @@ func (handler *Handler) registryConfigure(w http.ResponseWriter, r *http.Request
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	registry, err := handler.RegistryService.Registry(portainer.RegistryID(registryID))
+	registry, err := handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a registry with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -128,7 +128,7 @@ func (handler *Handler) registryConfigure(w http.ResponseWriter, r *http.Request
 		}
 	}
 
-	err = handler.RegistryService.UpdateRegistry(registry.ID, registry)
+	err = handler.DataStore.Registry().UpdateRegistry(registry.ID, registry)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist registry changes inside the database", err}
 	}

api/http/handler/registries/registry_create.go

@@ -55,7 +55,7 @@ func (handler *Handler) registryCreate(w http.ResponseWriter, r *http.Request) *
 		Gitlab:             payload.Gitlab,
 	}
 
-	err = handler.RegistryService.CreateRegistry(registry)
+	err = handler.DataStore.Registry().CreateRegistry(registry)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the registry inside the database", err}
 	}

api/http/handler/registries/registry_delete.go

@@ -16,14 +16,14 @@ func (handler *Handler) registryDelete(w http.ResponseWriter, r *http.Request) *
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry identifier route variable", err}
 	}
 
-	_, err = handler.RegistryService.Registry(portainer.RegistryID(registryID))
+	_, err = handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a registry with the specified identifier inside the database", err}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a registry with the specified identifier inside the database", err}
 	}
 
-	err = handler.RegistryService.DeleteRegistry(portainer.RegistryID(registryID))
+	err = handler.DataStore.Registry().DeleteRegistry(portainer.RegistryID(registryID))
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the registry from the database", err}
 	}

api/http/handler/registries/registry_inspect.go

@@ -16,7 +16,7 @@ func (handler *Handler) registryInspect(w http.ResponseWriter, r *http.Request)
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid registry identifier route variable", err}
 	}
 
-	registry, err := handler.RegistryService.Registry(portainer.RegistryID(registryID))
+	registry, err := handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a registry with the specified identifier inside the database", err}
 	} else if err != nil {

api/http/handler/registries/registry_list.go

@@ -10,7 +10,7 @@ import (
 
 // GET request on /api/registries
 func (handler *Handler) registryList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	registries, err := handler.RegistryService.Registries()
+	registries, err := handler.DataStore.Registry().Registries()
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}
 	}

api/http/handler/registries/registry_update.go

@@ -36,7 +36,7 @@ func (handler *Handler) registryUpdate(w http.ResponseWriter, r *http.Request) *
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	registry, err := handler.RegistryService.Registry(portainer.RegistryID(registryID))
+	registry, err := handler.DataStore.Registry().Registry(portainer.RegistryID(registryID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a registry with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -48,7 +48,7 @@ func (handler *Handler) registryUpdate(w http.ResponseWriter, r *http.Request) *
 	}
 
 	if payload.URL != nil {
-		registries, err := handler.RegistryService.Registries()
+		registries, err := handler.DataStore.Registry().Registries()
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve registries from the database", err}
 		}
@@ -88,7 +88,7 @@ func (handler *Handler) registryUpdate(w http.ResponseWriter, r *http.Request) *
 		registry.TeamAccessPolicies = payload.TeamAccessPolicies
 	}
 
-	err = handler.RegistryService.UpdateRegistry(registry.ID, registry)
+	err = handler.DataStore.Registry().UpdateRegistry(registry.ID, registry)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist registry changes inside the database", err}
 	}

api/http/handler/resourcecontrols/handler.go

@@ -12,7 +12,7 @@ import (
 // Handler is the HTTP handler used to handle resource control operations.
 type Handler struct {
 	*mux.Router
-	ResourceControlService portainer.ResourceControlService
+	DataStore portainer.DataStore
 }
 
 // NewHandler creates a handler to manage resource control operations.

api/http/handler/resourcecontrols/resourcecontrol_create.go

@@ -68,7 +68,7 @@ func (handler *Handler) resourceControlCreate(w http.ResponseWriter, r *http.Req
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid type value. Value must be one of: container, service, volume, network, secret, stack or config", portainer.ErrInvalidResourceControlType}
 	}
 
-	rc, err := handler.ResourceControlService.ResourceControlByResourceIDAndType(payload.ResourceID, resourceControlType)
+	rc, err := handler.DataStore.ResourceControl().ResourceControlByResourceIDAndType(payload.ResourceID, resourceControlType)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve resource controls from the database", err}
 	}
@@ -104,7 +104,7 @@ func (handler *Handler) resourceControlCreate(w http.ResponseWriter, r *http.Req
 		TeamAccesses:       teamAccesses,
 	}
 
-	err = handler.ResourceControlService.CreateResourceControl(&resourceControl)
+	err = handler.DataStore.ResourceControl().CreateResourceControl(&resourceControl)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist the resource control inside the database", err}
 	}

api/http/handler/resourcecontrols/resourcecontrol_delete.go

@@ -16,14 +16,14 @@ func (handler *Handler) resourceControlDelete(w http.ResponseWriter, r *http.Req
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid resource control identifier route variable", err}
 	}
 
-	_, err = handler.ResourceControlService.ResourceControl(portainer.ResourceControlID(resourceControlID))
+	_, err = handler.DataStore.ResourceControl().ResourceControl(portainer.ResourceControlID(resourceControlID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a resource control with the specified identifier inside the database", err}
 	} else if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to find a resource control with with the specified identifier inside the database", err}
 	}
 
-	err = handler.ResourceControlService.DeleteResourceControl(portainer.ResourceControlID(resourceControlID))
+	err = handler.DataStore.ResourceControl().DeleteResourceControl(portainer.ResourceControlID(resourceControlID))
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to remove the resource control from the database", err}
 	}

api/http/handler/resourcecontrols/resourcecontrol_update.go

@@ -42,7 +42,7 @@ func (handler *Handler) resourceControlUpdate(w http.ResponseWriter, r *http.Req
 		return &httperror.HandlerError{http.StatusBadRequest, "Invalid request payload", err}
 	}
 
-	resourceControl, err := handler.ResourceControlService.ResourceControl(portainer.ResourceControlID(resourceControlID))
+	resourceControl, err := handler.DataStore.ResourceControl().ResourceControl(portainer.ResourceControlID(resourceControlID))
 	if err == portainer.ErrObjectNotFound {
 		return &httperror.HandlerError{http.StatusNotFound, "Unable to find a resource control with the specified identifier inside the database", err}
 	} else if err != nil {
@@ -85,7 +85,7 @@ func (handler *Handler) resourceControlUpdate(w http.ResponseWriter, r *http.Req
 		return &httperror.HandlerError{http.StatusForbidden, "Permission denied to update the resource control", portainer.ErrResourceAccessDenied}
 	}
 
-	err = handler.ResourceControlService.UpdateResourceControl(resourceControl.ID, resourceControl)
+	err = handler.DataStore.ResourceControl().UpdateResourceControl(resourceControl.ID, resourceControl)
 	if err != nil {
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist resource control changes inside the database", err}
 	}