WIP: migrade data

api/cmd/portainer/main.go

@@ -23,6 +23,7 @@ import (
 	"github.com/portainer/portainer/api/database/boltdb"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/datastore/migrations"
 	"github.com/portainer/portainer/api/docker"
 	"github.com/portainer/portainer/api/exec"
 	"github.com/portainer/portainer/api/filesystem"
@@ -118,13 +119,20 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
 			logrus.Fatalf("Something Failed during creation of new database: %v", err)
 		}
 		if storedVersion != portainer.DBVersion {
-			err = store.MigrateData()
+			m := migrations.NewMigrator(*store)
+			err = m.Migrate(storedVersion)
 			if err != nil {
 				logrus.Fatalf("Failed migration: %v", err)
 			}
 		}
 	}
 
+	m := migrations.NewMigrator(*store)
+
+	m.Migrate(18)
+
+	logrus.Fatal("Dieing.....")
+
 	err = updateSettingsFromFlags(store, flags)
 	if err != nil {
 		log.Fatalf("Failed updating settings from flags: %v", err)

api/database/boltdb/db.go

@@ -407,6 +407,9 @@ func (connection *DbConnection) RestoreMetadata(s map[string]interface{}) error
 
 		err = connection.Batch(func(tx *bolt.Tx) error {
 			bucket := tx.Bucket([]byte(bucketName))
+			if bucket == nil {
+				return nil
+			}
 			return bucket.SetSequence(uint64(id))
 		})
 	}

api/datastore/migrations/1645580390_users_to_db_18.go

@@ -0,0 +1,53 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   17,
+		Timestamp: 1645580390,
+		Up:        v17_up_users_to_18,
+		Down:      v17_down_users_from_18,
+		Name:      "Users to 18",
+	})
+}
+
+func v17_up_users_to_18() error {
+	legacyUsers, err := migrator.store.UserService.Users()
+	if err != nil {
+		return err
+	}
+
+	for _, user := range legacyUsers {
+		user.PortainerAuthorizations = map[portainer.Authorization]bool{
+			portainer.OperationPortainerDockerHubInspect:        true,
+			portainer.OperationPortainerEndpointGroupList:       true,
+			portainer.OperationPortainerEndpointList:            true,
+			portainer.OperationPortainerEndpointInspect:         true,
+			portainer.OperationPortainerEndpointExtensionAdd:    true,
+			portainer.OperationPortainerEndpointExtensionRemove: true,
+			portainer.OperationPortainerExtensionList:           true,
+			portainer.OperationPortainerMOTD:                    true,
+			portainer.OperationPortainerRegistryList:            true,
+			portainer.OperationPortainerRegistryInspect:         true,
+			portainer.OperationPortainerTeamList:                true,
+			portainer.OperationPortainerTemplateList:            true,
+			portainer.OperationPortainerTemplateInspect:         true,
+			portainer.OperationPortainerUserList:                true,
+			portainer.OperationPortainerUserMemberships:         true,
+		}
+
+		err = migrator.store.UserService.UpdateUser(user.ID, &user)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func v17_down_users_from_18() error {
+	return nil
+}

api/datastore/migrations/1645580396_endpoints_to_db_18.go

@@ -0,0 +1,50 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   17,
+		Timestamp: 1645580396,
+		Up:        v17_up_endpoints_to_18,
+		Down:      v17_down_endpoints_from_18,
+		Name:      "Endpoints to 18",
+	})
+}
+
+func v17_up_endpoints_to_18() error {
+	legacyEndpoints, err := migrator.store.EndpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		endpoint.UserAccessPolicies = make(portainer.UserAccessPolicies)
+		for _, userID := range endpoint.AuthorizedUsers {
+			endpoint.UserAccessPolicies[userID] = portainer.AccessPolicy{
+				RoleID: 4,
+			}
+		}
+
+		endpoint.TeamAccessPolicies = make(portainer.TeamAccessPolicies)
+		for _, teamID := range endpoint.AuthorizedTeams {
+			endpoint.TeamAccessPolicies[teamID] = portainer.AccessPolicy{
+				RoleID: 4,
+			}
+		}
+
+		err = migrator.store.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func v17_down_endpoints_from_18() error {
+	return nil
+}

api/datastore/migrations/1645580400_endpoints_groups_to_db_18.go

@@ -0,0 +1,50 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   17,
+		Timestamp: 1645580400,
+		Up:        v17_up_endpoints_groups_to_18,
+		Down:      v17_down_endpoints_groups_from_18,
+		Name:      "Endpoints groups to 18",
+	})
+}
+
+func v17_up_endpoints_groups_to_18() error {
+	legacyEndpointGroups, err := migrator.store.EndpointGroupService.EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	for _, endpointGroup := range legacyEndpointGroups {
+		endpointGroup.UserAccessPolicies = make(portainer.UserAccessPolicies)
+		for _, userID := range endpointGroup.AuthorizedUsers {
+			endpointGroup.UserAccessPolicies[userID] = portainer.AccessPolicy{
+				RoleID: 4,
+			}
+		}
+
+		endpointGroup.TeamAccessPolicies = make(portainer.TeamAccessPolicies)
+		for _, teamID := range endpointGroup.AuthorizedTeams {
+			endpointGroup.TeamAccessPolicies[teamID] = portainer.AccessPolicy{
+				RoleID: 4,
+			}
+		}
+
+		err = migrator.store.EndpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func v17_down_endpoints_groups_from_18() error {
+	return nil
+}

api/datastore/migrations/1645580420_registries_to_db_18.go

@@ -0,0 +1,46 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   17,
+		Timestamp: 1645580420,
+		Up:        v17_up_registries_to_18,
+		Down:      v17_down_registries_to_18,
+		Name:      "Registries to 18",
+	})
+}
+
+func v17_up_registries_to_18() error {
+	legacyRegistries, err := migrator.store.RegistryService.Registries()
+	if err != nil {
+		return err
+	}
+
+	for _, registry := range legacyRegistries {
+		registry.UserAccessPolicies = make(portainer.UserAccessPolicies)
+		for _, userID := range registry.AuthorizedUsers {
+			registry.UserAccessPolicies[userID] = portainer.AccessPolicy{}
+		}
+
+		registry.TeamAccessPolicies = make(portainer.TeamAccessPolicies)
+		for _, teamID := range registry.AuthorizedTeams {
+			registry.TeamAccessPolicies[teamID] = portainer.AccessPolicy{}
+		}
+
+		err =  migrator.store.RegistryService.UpdateRegistry(registry.ID, &registry)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func v17_down_registries_to_18() error {
+	return nil
+}

api/datastore/migrations/1645677508_settings_to_db_19.go

@@ -0,0 +1,33 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   18,
+		Timestamp: 1645677508,
+		Up:        v18_up_settings_to_db_19,
+		Down:      v18_down_settings_to_db_19,
+		Name:      "settings to db 19",
+	})
+}
+
+func v18_up_settings_to_db_19() error {
+	legacySettings, err := migrator.store.SettingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	if legacySettings.EdgeAgentCheckinInterval == 0 {
+		legacySettings.EdgeAgentCheckinInterval = portainer.DefaultEdgeAgentCheckinIntervalInSeconds
+	}
+
+	return migrator.store.SettingsService.UpdateSettings(legacySettings)
+}
+
+func v18_down_settings_to_db_19() error {
+	return nil
+}

api/datastore/migrations/1645736810_users_to_db_20.go

@@ -0,0 +1,23 @@
+package migrations
+
+import (
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   19,
+		Timestamp: 1645736810,
+		Up:        v19_up_users_to_db_20,
+		Down:      v19_down_users_to_db_20,
+		Name:      "users to db 20",
+	})
+}
+
+func v19_up_users_to_db_20() error {
+	return migrator.store.AuthorizationService.UpdateUsersAuthorizations()
+}
+
+func v19_down_users_to_db_20() error {
+	return nil
+}

api/datastore/migrations/1645737700_settings_to_db_20.go

@@ -0,0 +1,30 @@
+package migrations
+
+import (
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   19,
+		Timestamp: 1645737700,
+		Up:        v19_up_settings_to_db_20,
+		Down:      v19_down_settings_to_db_20,
+		Name:      "settings to db 20",
+	})
+}
+
+func v19_up_settings_to_db_20() error {
+	legacySettings, err := migrator.store.SettingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.AllowVolumeBrowserForRegularUsers = false
+
+	return migrator.store.SettingsService.UpdateSettings(legacySettings)
+}
+
+func v19_down_settings_to_db_20() error {
+	return nil
+}

api/datastore/migrations/1645737802_schedules_to_db_20.go

@@ -0,0 +1,58 @@
+package migrations
+
+import (
+	"strings"
+
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+const scheduleScriptExecutionJobType = 1
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   19,
+		Timestamp: 1645737802,
+		Up:        v19_up_schedules_to_db_20,
+		Down:      v19_down_schedules_to_db_20,
+		Name:      "schedules to db 20",
+	})
+}
+
+func v19_up_schedules_to_db_20() error {
+	legacySchedules, err := migrator.store.ScheduleService.Schedules()
+	if err != nil {
+		return err
+	}
+
+	for _, schedule := range legacySchedules {
+		if schedule.JobType == scheduleScriptExecutionJobType {
+			if schedule.CronExpression == "0 0 * * *" {
+				schedule.CronExpression = "0 * * * *"
+			} else if schedule.CronExpression == "0 0 0/2 * *" {
+				schedule.CronExpression = "0 */2 * * *"
+			} else if schedule.CronExpression == "0 0 0 * *" {
+				schedule.CronExpression = "0 0 * * *"
+			} else {
+				revisedCronExpression := strings.Split(schedule.CronExpression, " ")
+				if len(revisedCronExpression) == 5 {
+					continue
+				}
+
+				revisedCronExpression = revisedCronExpression[1:]
+				schedule.CronExpression = strings.Join(revisedCronExpression, " ")
+			}
+
+			err := migrator.store.ScheduleService.UpdateSchedule(schedule.ID, &schedule)
+			if err != nil {
+				return err
+			}
+		}
+
+	}
+
+	return nil
+}
+
+func v19_down_schedules_to_db_20() error {
+	return nil
+}

api/datastore/migrations/1646090591_resource_control_to_22.go

@@ -0,0 +1,37 @@
+package migrations
+
+import (
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   20,
+		Timestamp: 1646090591,
+		Up:        v20_up_resource_control_to_22,
+		Down:      v20_down_resource_control_to_22,
+		Name:      "resource control to 22",
+	})
+}
+
+func v20_up_resource_control_to_22() error {
+	legacyResourceControls, err := migrator.store.ResourceControlService.ResourceControls()
+	if err != nil {
+		return err
+	}
+
+	for _, resourceControl := range legacyResourceControls {
+		resourceControl.AdministratorsOnly = false
+
+		err := migrator.store.ResourceControlService.UpdateResourceControl(resourceControl.ID, &resourceControl)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func v20_down_resource_control_to_22() error {
+	return nil
+}

api/datastore/migrations/1646090646_user_and_roles_to_22.go

@@ -0,0 +1,82 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+	"github.com/portainer/portainer/api/internal/authorization"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   20,
+		Timestamp: 1646090646,
+		Up:        v20_up_user_and_roles_to_22,
+		Down:      v20_down_user_and_roles_to_22,
+		Name:      "user and roles to 22",
+	})
+}
+
+func v20_up_user_and_roles_to_22() error {
+	legacyUsers, err := migrator.store.UserService.Users()
+	if err != nil {
+		return err
+	}
+
+	settings, err := migrator.store.SettingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	for _, user := range legacyUsers {
+		user.PortainerAuthorizations = authorization.DefaultPortainerAuthorizations()
+		err = migrator.store.UserService.UpdateUser(user.ID, &user)
+		if err != nil {
+			return err
+		}
+	}
+
+	endpointAdministratorRole, err := migrator.store.RoleService.Role(portainer.RoleID(1))
+	if err != nil {
+		return err
+	}
+	endpointAdministratorRole.Priority = 1
+	endpointAdministratorRole.Authorizations = authorization.DefaultEndpointAuthorizationsForEndpointAdministratorRole()
+
+	err = migrator.store.RoleService.UpdateRole(endpointAdministratorRole.ID, endpointAdministratorRole)
+
+	helpDeskRole, err := migrator.store.RoleService.Role(portainer.RoleID(2))
+	if err != nil {
+		return err
+	}
+	helpDeskRole.Priority = 2
+	helpDeskRole.Authorizations = authorization.DefaultEndpointAuthorizationsForHelpDeskRole(settings.AllowVolumeBrowserForRegularUsers)
+
+	err = migrator.store.RoleService.UpdateRole(helpDeskRole.ID, helpDeskRole)
+
+	standardUserRole, err := migrator.store.RoleService.Role(portainer.RoleID(3))
+	if err != nil {
+		return err
+	}
+	standardUserRole.Priority = 3
+	standardUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForStandardUserRole(settings.AllowVolumeBrowserForRegularUsers)
+
+	err = migrator.store.RoleService.UpdateRole(standardUserRole.ID, standardUserRole)
+
+	readOnlyUserRole, err := migrator.store.RoleService.Role(portainer.RoleID(4))
+	if err != nil {
+		return err
+	}
+	readOnlyUserRole.Priority = 4
+	readOnlyUserRole.Authorizations = authorization.DefaultEndpointAuthorizationsForReadOnlyUserRole(settings.AllowVolumeBrowserForRegularUsers)
+
+	err = migrator.store.RoleService.UpdateRole(readOnlyUserRole.ID, readOnlyUserRole)
+	if err != nil {
+		return err
+	}
+
+	return migrator.store.AuthorizationService.UpdateUsersAuthorizations()
+}
+
+func v20_down_user_and_roles_to_22() error {
+	return nil
+}

api/datastore/migrations/1646090838_tags_to_23.go

@@ -0,0 +1,39 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+	"github.com/sirupsen/logrus"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   22,
+		Timestamp: 1646090838,
+		Up:        v22_up_tags_to_23,
+		Down:      v22_down_tags_to_23,
+		Name:      "tags to 23",
+	})
+}
+
+func v22_up_tags_to_23() error {
+	logrus.Info("Updating tags")
+	tags, err := migrator.store.TagService.Tags()
+	if err != nil {
+		return err
+	}
+
+	for _, tag := range tags {
+		tag.EndpointGroups = make(map[portainer.EndpointGroupID]bool)
+		tag.Endpoints = make(map[portainer.EndpointID]bool)
+		err = migrator.store.TagService.UpdateTag(tag.ID, &tag)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func v22_down_tags_to_23() error {
+	return nil
+}

api/datastore/migrations/1646091011_endpoints_and_endpoint_groups_to_23.go

@@ -0,0 +1,94 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+	"github.com/sirupsen/logrus"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   22,
+		Timestamp: 1646091011,
+		Up:        v22_up_endpoints_and_endpoint_groups_to_23,
+		Down:      v22_down_endpoints_and_endpoint_groups_to_23,
+		Name:      "endpoints and endpoint groups to 23",
+	})
+}
+
+func v22_up_endpoints_and_endpoint_groups_to_23() error {
+	logrus.Info("Updating endpoints and endpoint groups")
+	tags, err := migrator.store.TagService.Tags()
+	if err != nil {
+		return err
+	}
+
+	tagsNameMap := make(map[string]portainer.Tag)
+	for _, tag := range tags {
+		tagsNameMap[tag.Name] = tag
+	}
+
+	endpoints, err := migrator.store.EndpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		endpointTags := make([]portainer.TagID, 0)
+		for _, tagName := range endpoint.Tags {
+			tag, ok := tagsNameMap[tagName]
+			if ok {
+				endpointTags = append(endpointTags, tag.ID)
+				tag.Endpoints[endpoint.ID] = true
+			}
+		}
+		endpoint.TagIDs = endpointTags
+		err = migrator.store.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+
+		relation := &portainer.EndpointRelation{
+			EndpointID: endpoint.ID,
+			EdgeStacks: map[portainer.EdgeStackID]bool{},
+		}
+
+		err = migrator.store.EndpointRelationService.Create(relation)
+		if err != nil {
+			return err
+		}
+	}
+
+	endpointGroups, err := migrator.store.EndpointGroupService.EndpointGroups()
+	if err != nil {
+		return err
+	}
+
+	for _, endpointGroup := range endpointGroups {
+		endpointGroupTags := make([]portainer.TagID, 0)
+		for _, tagName := range endpointGroup.Tags {
+			tag, ok := tagsNameMap[tagName]
+			if ok {
+				endpointGroupTags = append(endpointGroupTags, tag.ID)
+				tag.EndpointGroups[endpointGroup.ID] = true
+			}
+		}
+		endpointGroup.TagIDs = endpointGroupTags
+		err = migrator.store.EndpointGroupService.UpdateEndpointGroup(endpointGroup.ID, &endpointGroup)
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, tag := range tagsNameMap {
+		err = migrator.store.TagService.UpdateTag(tag.ID, &tag)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func v22_down_endpoints_and_endpoint_groups_to_23() error {
+	return nil
+}

api/datastore/migrations/1646091296_settings_to_24.go

@@ -0,0 +1,41 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+	"github.com/sirupsen/logrus"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   23,
+		Timestamp: 1646091296,
+		Up:        v23_up_settings_to_24,
+		Down:      v23_down_settings_to_24,
+		Name:      "settings to 25",
+	})
+}
+
+func v23_up_settings_to_24() error {
+	logrus.Info("Updating settings")
+
+	legacySettings, err := migrator.store.SettingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	if legacySettings.TemplatesURL == "" {
+		legacySettings.TemplatesURL = portainer.DefaultTemplatesURL
+	}
+
+	legacySettings.UserSessionTimeout = portainer.DefaultUserSessionTimeout
+	legacySettings.EnableTelemetry = true
+
+	legacySettings.AllowContainerCapabilitiesForRegularUsers = true
+
+	return migrator.store.SettingsService.UpdateSettings(legacySettings)
+}
+
+func v23_down_settings_to_24() error {
+	return nil
+}

api/datastore/migrations/1646095944_endpoint_settings_to_25.go

@@ -0,0 +1,68 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+	"github.com/sirupsen/logrus"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   24,
+		Timestamp: 1646095944,
+		Up:        v24_up_endpoint_settings_to_25,
+		Down:      v24_down_endpoint_settings_to_25,
+		Name:      "endpoint settings to 25",
+	})
+}
+
+func v24_up_endpoint_settings_to_25() error {
+	logrus.Info("Updating endpoint settings")
+	settings, err := migrator.store.SettingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	endpoints, err := migrator.store.EndpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for i := range endpoints {
+		endpoint := endpoints[i]
+
+		securitySettings := portainer.EndpointSecuritySettings{}
+
+		if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment ||
+			endpoint.Type == portainer.AgentOnDockerEnvironment ||
+			endpoint.Type == portainer.DockerEnvironment {
+
+			securitySettings = portainer.EndpointSecuritySettings{
+				AllowBindMountsForRegularUsers:            settings.AllowBindMountsForRegularUsers,
+				AllowContainerCapabilitiesForRegularUsers: settings.AllowContainerCapabilitiesForRegularUsers,
+				AllowDeviceMappingForRegularUsers:         settings.AllowDeviceMappingForRegularUsers,
+				AllowHostNamespaceForRegularUsers:         settings.AllowHostNamespaceForRegularUsers,
+				AllowPrivilegedModeForRegularUsers:        settings.AllowPrivilegedModeForRegularUsers,
+				AllowStackManagementForRegularUsers:       settings.AllowStackManagementForRegularUsers,
+			}
+
+			if endpoint.Type == portainer.AgentOnDockerEnvironment || endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
+				securitySettings.AllowVolumeBrowserForRegularUsers = settings.AllowVolumeBrowserForRegularUsers
+				securitySettings.EnableHostManagementFeatures = settings.EnableHostManagementFeatures
+			}
+		}
+
+		endpoint.SecuritySettings = securitySettings
+
+		err = migrator.store.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func v24_down_endpoint_settings_to_25() error {
+	return nil
+}

api/datastore/migrations/1646096711_stack_resource_control_to_27.go

@@ -0,0 +1,57 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+	"github.com/portainer/portainer/api/internal/stackutils"
+	"github.com/sirupsen/logrus"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   26,
+		Timestamp: 1646096711,
+		Up:        v26_up_stack_resource_control_to_27,
+		Down:      v26_down_stack_resource_control_to_27,
+		Name:      "stack resource control to 27",
+	})
+}
+
+func v26_up_stack_resource_control_to_27() error {
+	logrus.Info("Updating stack resource controls")
+	resourceControls, err := migrator.store.ResourceControlService.ResourceControls()
+	if err != nil {
+		return err
+	}
+
+	for _, resource := range resourceControls {
+		if resource.Type != portainer.StackResourceControl {
+			continue
+		}
+
+		stackName := resource.ResourceID
+
+		stack, err := migrator.store.StackService.StackByName(stackName)
+		if err != nil {
+			if err == errors.ErrObjectNotFound {
+				continue
+			}
+
+			return err
+		}
+
+		resource.ResourceID = stackutils.ResourceControlID(stack.EndpointID, stack.Name)
+
+		err = migrator.store.ResourceControlService.UpdateResourceControl(resource.ID, &resource)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func v26_down_stack_resource_control_to_27() error {
+	return nil
+}

api/datastore/migrations/1646096869_settings_to_30.go

@@ -0,0 +1,33 @@
+package migrations
+
+import (
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   29,
+		Timestamp: 1646096869,
+		Up:        v29_up_settings_to_30,
+		Down:      v29_down_settings_to_30,
+		Name:      "settings to 30",
+	})
+}
+
+// so setting to false and "", is what would happen without this code
+// I'm going to bet there's zero point to changing the value inthe DB
+// Public for testing
+func v29_up_settings_to_30() error {
+	legacySettings, err := migrator.store.SettingsService.Settings()
+	if err != nil {
+		return err
+	}
+
+	legacySettings.OAuthSettings.SSO = false
+	legacySettings.OAuthSettings.LogoutURI = ""
+	return migrator.store.SettingsService.UpdateSettings(legacySettings)
+}
+
+func v29_down_settings_to_30() error {
+	return nil
+}

api/datastore/migrations/1646097709_registries_to_32.go

@@ -0,0 +1,62 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   31,
+		Timestamp: 1646097709,
+		Up:        v31_up_registries_to_32,
+		Down:      v31_down_registries_to_32,
+		Name:      "registries to 32",
+	})
+}
+
+func v31_up_registries_to_32() error {
+	registries, err := migrator.store.RegistryService.Registries()
+	if err != nil {
+		return err
+	}
+
+	endpoints, err := migrator.store.EndpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, registry := range registries {
+
+		registry.RegistryAccesses = portainer.RegistryAccesses{}
+
+		for _, endpoint := range endpoints {
+
+			filteredUserAccessPolicies := portainer.UserAccessPolicies{}
+			for userId, registryPolicy := range registry.UserAccessPolicies {
+				if _, found := endpoint.UserAccessPolicies[userId]; found {
+					filteredUserAccessPolicies[userId] = registryPolicy
+				}
+			}
+
+			filteredTeamAccessPolicies := portainer.TeamAccessPolicies{}
+			for teamId, registryPolicy := range registry.TeamAccessPolicies {
+				if _, found := endpoint.TeamAccessPolicies[teamId]; found {
+					filteredTeamAccessPolicies[teamId] = registryPolicy
+				}
+			}
+
+			registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{
+				UserAccessPolicies: filteredUserAccessPolicies,
+				TeamAccessPolicies: filteredTeamAccessPolicies,
+				Namespaces:         []string{},
+			}
+		}
+		migrator.store.RegistryService.UpdateRegistry(registry.ID, &registry)
+	}
+	return nil
+}
+
+func v31_down_registries_to_32() error {
+	return nil
+}

api/datastore/migrations/1646097896_dockerhub_to_32.go

@@ -0,0 +1,109 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   31,
+		Timestamp: 1646097896,
+		Up:        v31_up_dockerhub_to_32,
+		Down:      v31_down_dockerhub_to_32,
+		Name:      "dockerhub to 32",
+	})
+}
+
+func v31_up_dockerhub_to_32() error {
+	dockerhub, err := migrator.store.DockerHubService.DockerHub()
+	if err == errors.ErrObjectNotFound {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	if !dockerhub.Authentication {
+		return nil
+	}
+
+	registry := &portainer.Registry{
+		Type:             portainer.DockerHubRegistry,
+		Name:             "Dockerhub (authenticated - migrated)",
+		URL:              "docker.io",
+		Authentication:   true,
+		Username:         dockerhub.Username,
+		Password:         dockerhub.Password,
+		RegistryAccesses: portainer.RegistryAccesses{},
+	}
+
+	// The following code will make this function idempotent.
+	// i.e. if run again, it will not change the data.  It will ensure that
+	// we only have one migrated registry entry. Duplicates will be removed
+	// if they exist and which has been happening due to earlier migration bugs
+	migrated := false
+	registries, _ := migrator.store.RegistryService.Registries()
+	for _, r := range registries {
+		if r.Type == registry.Type &&
+			r.Name == registry.Name &&
+			r.URL == registry.URL &&
+			r.Authentication == registry.Authentication {
+
+			if !migrated {
+				// keep this one entry
+				migrated = true
+			} else {
+				// delete subsequent duplicates
+				migrator.store.RegistryService.DeleteRegistry(portainer.RegistryID(r.ID))
+			}
+		}
+	}
+
+	if migrated {
+		return nil
+	}
+
+	endpoints, err := migrator.store.EndpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+
+		if endpoint.Type != portainer.KubernetesLocalEnvironment &&
+			endpoint.Type != portainer.AgentOnKubernetesEnvironment &&
+			endpoint.Type != portainer.EdgeAgentOnKubernetesEnvironment {
+
+			userAccessPolicies := portainer.UserAccessPolicies{}
+			for userId := range endpoint.UserAccessPolicies {
+				if _, found := endpoint.UserAccessPolicies[userId]; found {
+					userAccessPolicies[userId] = portainer.AccessPolicy{
+						RoleID: 0,
+					}
+				}
+			}
+
+			teamAccessPolicies := portainer.TeamAccessPolicies{}
+			for teamId := range endpoint.TeamAccessPolicies {
+				if _, found := endpoint.TeamAccessPolicies[teamId]; found {
+					teamAccessPolicies[teamId] = portainer.AccessPolicy{
+						RoleID: 0,
+					}
+				}
+			}
+
+			registry.RegistryAccesses[endpoint.ID] = portainer.RegistryAccessPolicies{
+				UserAccessPolicies: userAccessPolicies,
+				TeamAccessPolicies: teamAccessPolicies,
+				Namespaces:         []string{},
+			}
+		}
+	}
+
+	return migrator.store.RegistryService.Create(registry)
+}
+
+func v31_down_dockerhub_to_32() error {
+	return nil
+}

api/datastore/migrations/1646097916_volume_resource_control_to_32.go

@@ -0,0 +1,117 @@
+package migrations
+
+import (
+	"fmt"
+	"log"
+
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+	"github.com/portainer/portainer/api/internal/endpointutils"
+	snapshotutils "github.com/portainer/portainer/api/internal/snapshot"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   31,
+		Timestamp: 1646097916,
+		Up:        v31_up_volume_resource_control_to_32,
+		Down:      v31_down_volume_resource_control_to_32,
+		Name:      "volume resource control to 32",
+	})
+}
+
+func findResourcesToUpdateForDB32(dockerID string, volumesData map[string]interface{}, toUpdate map[portainer.ResourceControlID]string, volumeResourceControls map[string]*portainer.ResourceControl) {
+	volumes := volumesData["Volumes"].([]interface{})
+	for _, volumeMeta := range volumes {
+		volume := volumeMeta.(map[string]interface{})
+		volumeName, nameExist := volume["Name"].(string)
+		if !nameExist {
+			continue
+		}
+		createTime, createTimeExist := volume["CreatedAt"].(string)
+		if !createTimeExist {
+			continue
+		}
+
+		oldResourceID := fmt.Sprintf("%s%s", volumeName, createTime)
+		resourceControl, ok := volumeResourceControls[oldResourceID]
+
+		if ok {
+			toUpdate[resourceControl.ID] = fmt.Sprintf("%s_%s", volumeName, dockerID)
+		}
+	}
+}
+
+func v31_up_volume_resource_control_to_32() error {
+	endpoints, err := migrator.store.EndpointService.Endpoints()
+	if err != nil {
+		return fmt.Errorf("failed fetching environments: %w", err)
+	}
+
+	resourceControls, err := migrator.store.ResourceControlService.ResourceControls()
+	if err != nil {
+		return fmt.Errorf("failed fetching resource controls: %w", err)
+	}
+
+	toUpdate := map[portainer.ResourceControlID]string{}
+	volumeResourceControls := map[string]*portainer.ResourceControl{}
+
+	for i := range resourceControls {
+		resourceControl := resourceControls[i]
+		if resourceControl.Type == portainer.VolumeResourceControl {
+			volumeResourceControls[resourceControl.ResourceID] = &resourceControl
+		}
+	}
+
+	for _, endpoint := range endpoints {
+		if !endpointutils.IsDockerEndpoint(&endpoint) {
+			continue
+		}
+
+		totalSnapshots := len(endpoint.Snapshots)
+		if totalSnapshots == 0 {
+			log.Println("[DEBUG] [volume migration] [message: no snapshot found]")
+			continue
+		}
+
+		snapshot := endpoint.Snapshots[totalSnapshots-1]
+
+		endpointDockerID, err := snapshotutils.FetchDockerID(snapshot)
+		if err != nil {
+			log.Printf("[WARN] [database,migrator,v31] [message: failed fetching environment docker id] [err: %s]", err)
+			continue
+		}
+
+		if volumesData, done := snapshot.SnapshotRaw.Volumes.(map[string]interface{}); done {
+			if volumesData["Volumes"] == nil {
+				log.Println("[DEBUG] [volume migration] [message: no volume data found]")
+				continue
+			}
+
+			findResourcesToUpdateForDB32(endpointDockerID, volumesData, toUpdate, volumeResourceControls)
+		}
+	}
+
+	for _, resourceControl := range volumeResourceControls {
+		if newResourceID, ok := toUpdate[resourceControl.ID]; ok {
+			resourceControl.ResourceID = newResourceID
+			err := migrator.store.ResourceControlService.UpdateResourceControl(resourceControl.ID, resourceControl)
+			if err != nil {
+				return fmt.Errorf("failed updating resource control %d: %w", resourceControl.ID, err)
+			}
+
+		} else {
+			err := migrator.store.ResourceControlService.DeleteResourceControl(resourceControl.ID)
+			if err != nil {
+				return fmt.Errorf("failed deleting resource control %d: %w", resourceControl.ID, err)
+			}
+			log.Printf("[DEBUG] [volume migration] [message: legacy resource control(%s) has been deleted]", resourceControl.ResourceID)
+		}
+	}
+
+	return nil
+}
+
+func v31_down_volume_resource_control_to_32() error {
+	return nil
+}

api/datastore/migrations/1646097944_kubeconfig_expiry_to_32.go

@@ -0,0 +1,29 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   31,
+		Timestamp: 1646097944,
+		Up:        v31_up_kubeconfig_expiry_to_32,
+		Down:      v31_down_kubeconfig_expiry_to_32,
+		Name:      "kubeconfig expiry to 32",
+	})
+}
+
+func v31_up_kubeconfig_expiry_to_32() error {
+	settings, err := migrator.store.SettingsService.Settings()
+	if err != nil {
+		return err
+	}
+	settings.KubeconfigExpiry = portainer.DefaultKubeconfigExpiry
+	return migrator.store.SettingsService.UpdateSettings(settings)
+}
+
+func v31_down_kubeconfig_expiry_to_32() error {
+	return nil
+}

api/datastore/migrations/1646097962_helm_repo_url_to_32.go

@@ -0,0 +1,29 @@
+package migrations
+
+import (
+	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   31,
+		Timestamp: 1646097962,
+		Up:        v31_up_helm_repo_url_to_32,
+		Down:      v31_down_helm_repo_url_to_32,
+		Name:      "helm repo url to 32",
+	})
+}
+
+func v31_up_helm_repo_url_to_32() error {
+	settings, err := migrator.store.SettingsService.Settings()
+	if err != nil {
+		return err
+	}
+	settings.HelmRepositoryURL = portainer.DefaultHelmRepositoryURL
+	return migrator.store.SettingsService.UpdateSettings(settings)
+}
+
+func v31_down_helm_repo_url_to_32() error {
+	return nil
+}

api/datastore/migrations/migration.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+die () {
+    echo >&2 "$@"
+    exit 1
+}
+
+[ "$#" -eq 2 ] || die "Usage - version \"space separated context\""
+
+TIMESTAMP=$(date +%s)
+VERSION=$1
+CONTEXT=$2
+
+CONTEXT_SLUG="${CONTEXT// /_}"
+
+cat << EOF >${TIMESTAMP}_${CONTEXT_SLUG}.go
+package migrations
+
+import (
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+)
+
+func init() {
+	migrator.AddMigration(types.Migration{
+		Version:   ${VERSION},
+		Timestamp: ${TIMESTAMP},
+		Up:        v${VERSION}_up_${CONTEXT_SLUG},
+		Down:      v${VERSION}_down_${CONTEXT_SLUG},
+		Name:      "${CONTEXT}",
+	})
+}
+
+func v${VERSION}_up_${CONTEXT_SLUG}() error {
+	return nil
+}
+
+func v${VERSION}_down_${CONTEXT_SLUG}() error {
+	return nil
+}
+EOF

api/datastore/migrations/migrator.go

@@ -0,0 +1,111 @@
+package migrations
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/pkg/errors"
+	"github.com/portainer/portainer/api/datastore"
+	"github.com/portainer/portainer/api/datastore/migrations/types"
+	"github.com/sirupsen/logrus"
+)
+
+type apiDBVersion struct {
+	api string
+	db  int
+}
+type apiVersionsMap []apiDBVersion
+
+type migrations []*types.Migration
+
+type Migrator struct {
+	store      datastore.Store
+	Versions   []int
+	Migrations map[int]migrations
+}
+
+var migrator *Migrator = &Migrator{
+	Versions:   []int{},
+	Migrations: map[int]migrations{},
+}
+
+var versionsMap apiVersionsMap = apiVersionsMap{
+	{"2.12.0", 36},
+	{"2.9.3", 35},
+	{"2.10.0", 34},
+	{"2.9.2", 33},
+	{"2.9.1", 33},
+	{"2.9.0", 32},
+	{"2.7.0", 31},
+	{"2.6.0", 30},
+	{"2.4.0", 29},
+	{"2.4.0", 28},
+	{"2.2.0", 27},
+	{"2.1.0", 26},
+}
+
+func NewMigrator(m datastore.Store) *Migrator {
+	migrator.store = m
+	return migrator
+}
+
+func (m *Migrator) AddMigration(mg types.Migration) {
+	// Add the migration to the hash with version as key
+	if m.Migrations[mg.Version] == nil {
+		m.Migrations[mg.Version] = make(migrations, 0)
+	}
+	m.Migrations[mg.Version] = append(m.Migrations[mg.Version], &mg)
+
+	if !contains(m.Versions, mg.Version) {
+		// Insert version into versions array using insertion sort
+		index := 0
+		for index < len(m.Versions) {
+			if m.Versions[index] > mg.Version {
+				break
+			}
+			index++
+		}
+
+		m.Versions = append(m.Versions, mg.Version)
+		copy(m.Versions[index+1:], m.Versions[index:])
+		m.Versions[index] = mg.Version
+	}
+}
+
+func (m *Migrator) Migrate(currentVersion int) error {
+    migrationsToRun := &Migrator{
+        Versions:   []int{},
+        Migrations: map[int]migrations{},
+    }
+	for _, v := range m.Versions {
+		mg := m.Migrations[v]
+        // if migration version is below current version
+        if v < currentVersion {
+            continue
+        }
+        migrationsToRun.Versions = append(migrationsToRun.Versions, v)
+        migrationsToRun.Migrations[v] = mg
+	}
+
+    // TODO: Sort by Timestamp
+    for _, v := range migrationsToRun.Versions {
+        mg := m.Migrations[v]
+        for _, m := range mg {
+            logger := logrus.WithFields(logrus.Fields{"version": m.Version, "migration": m.Name})
+            logger.Info("starting migration")
+            err := m.Up()
+            if err != nil {
+                return errors.Wrap(err, fmt.Sprintf("while running migration for version %d, name `%s`", m.Version, m.Name))
+            }
+            m.Completed = true
+            logger.Info("migration completed successfully")
+        }
+    }
+	return nil
+}
+
+// TODO: move to utils
+func contains(s []int, searchterm int) bool {
+	i := sort.SearchInts(s, searchterm)
+	return i < len(s) && s[i] == searchterm
+}

api/datastore/migrations/types/types.go

@@ -0,0 +1,10 @@
+package types
+
+type Migration struct {
+	Version   int
+	Up        func() error
+	Down      func() error
+	Completed bool
+	Timestamp int32
+	Name      string
+}

api/datastore/services.go

@@ -33,6 +33,7 @@ import (
 	"github.com/portainer/portainer/api/dataservices/user"
 	"github.com/portainer/portainer/api/dataservices/version"
 	"github.com/portainer/portainer/api/dataservices/webhook"
+	"github.com/portainer/portainer/api/internal/authorization"
 	"github.com/sirupsen/logrus"
 )
 
@@ -68,6 +69,8 @@ type Store struct {
 	UserService               *user.Service
 	VersionService            *version.Service
 	WebhookService            *webhook.Service
+
+	AuthorizationService *authorization.Service // TODO: validate why it is not part of store
 }
 
 func (store *Store) initServices() error {
@@ -227,6 +230,13 @@ func (store *Store) initServices() error {
 	}
 	store.ScheduleService = scheduleService
 
+	authService := authorization.NewService(store)
+	if err != nil {
+		return err
+	}
+	store.AuthorizationService = authService
+
+
 	return nil
 }