diff --git a/cmd/flamenco-manager-poc/main.go b/cmd/flamenco-manager-poc/main.go
index 401e864a..5f0afcb1 100644
--- a/cmd/flamenco-manager-poc/main.go
+++ b/cmd/flamenco-manager-poc/main.go
@@ -37,6 +37,7 @@ import (
"gitlab.com/blender/flamenco-ng-poc/internal/appinfo"
"gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl"
+ "gitlab.com/blender/flamenco-ng-poc/internal/manager/config"
"gitlab.com/blender/flamenco-ng-poc/internal/manager/job_compilers"
"gitlab.com/blender/flamenco-ng-poc/internal/manager/persistence"
"gitlab.com/blender/flamenco-ng-poc/internal/manager/swagger_ui"
@@ -58,6 +59,10 @@ func main() {
if cliArgs.version {
return
}
+
+ // Load configuration.
+ configService := config.NewService()
+
if cliArgs.initDB {
log.Info().Msg("creating databases")
err := persistence.InitialSetup()
@@ -88,7 +93,7 @@ func main() {
log.Fatal().Err(err).Msg("error loading job compilers")
}
logStorage := task_logs.NewStorage("./task-logs") // TODO: load job storage path from configuration.
- flamenco := api_impl.NewFlamenco(compiler, persist, logStorage)
+ flamenco := api_impl.NewFlamenco(compiler, persist, logStorage, configService)
e := buildWebService(flamenco, persist)
// Start the web server.
diff --git a/internal/manager/api_impl/api_impl.go b/internal/manager/api_impl/api_impl.go
index 5ea37a53..c497f7a3 100644
--- a/internal/manager/api_impl/api_impl.go
+++ b/internal/manager/api_impl/api_impl.go
@@ -36,10 +36,11 @@ type Flamenco struct {
jobCompiler JobCompiler
persist PersistenceService
logStorage LogStorage
+ config ConfigService
}
// Generate mock implementations of these interfaces.
-//go:generate go run github.com/golang/mock/mockgen -destination mocks/api_impl_mock.gen.go -package mocks gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl PersistenceService,JobCompiler,LogStorage
+//go:generate go run github.com/golang/mock/mockgen -destination mocks/api_impl_mock.gen.go -package mocks gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl PersistenceService,JobCompiler,LogStorage,ConfigService
type PersistenceService interface {
StoreAuthoredJob(ctx context.Context, authoredJob job_compilers.AuthoredJob) error
@@ -68,14 +69,19 @@ type LogStorage interface {
RotateFile(logger zerolog.Logger, jobID, taskID string)
}
+type ConfigService interface {
+ ExpandVariables(valueToExpand, audience, platform string) string
+}
+
var _ api.ServerInterface = (*Flamenco)(nil)
// NewFlamenco creates a new Flamenco service, using the given JobCompiler.
-func NewFlamenco(jc JobCompiler, jps PersistenceService, ls LogStorage) *Flamenco {
+func NewFlamenco(jc JobCompiler, jps PersistenceService, ls LogStorage, cs ConfigService) *Flamenco {
return &Flamenco{
jobCompiler: jc,
persist: jps,
logStorage: ls,
+ config: cs,
}
}
diff --git a/internal/manager/api_impl/mocks/api_impl_mock.gen.go b/internal/manager/api_impl/mocks/api_impl_mock.gen.go
index d505e61d..c6ab90f1 100644
--- a/internal/manager/api_impl/mocks/api_impl_mock.gen.go
+++ b/internal/manager/api_impl/mocks/api_impl_mock.gen.go
@@ -1,5 +1,5 @@
// Code generated by MockGen. DO NOT EDIT.
-// Source: gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl (interfaces: PersistenceService,JobCompiler,LogStorage)
+// Source: gitlab.com/blender/flamenco-ng-poc/internal/manager/api_impl (interfaces: PersistenceService,JobCompiler,LogStorage,ConfigService)
// Package mocks is a generated GoMock package.
package mocks
@@ -268,3 +268,40 @@ func (mr *MockLogStorageMockRecorder) Write(arg0, arg1, arg2, arg3 interface{})
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockLogStorage)(nil).Write), arg0, arg1, arg2, arg3)
}
+
+// MockConfigService is a mock of ConfigService interface.
+type MockConfigService struct {
+ ctrl *gomock.Controller
+ recorder *MockConfigServiceMockRecorder
+}
+
+// MockConfigServiceMockRecorder is the mock recorder for MockConfigService.
+type MockConfigServiceMockRecorder struct {
+ mock *MockConfigService
+}
+
+// NewMockConfigService creates a new mock instance.
+func NewMockConfigService(ctrl *gomock.Controller) *MockConfigService {
+ mock := &MockConfigService{ctrl: ctrl}
+ mock.recorder = &MockConfigServiceMockRecorder{mock}
+ return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockConfigService) EXPECT() *MockConfigServiceMockRecorder {
+ return m.recorder
+}
+
+// ExpandVariables mocks base method.
+func (m *MockConfigService) ExpandVariables(arg0, arg1, arg2 string) string {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "ExpandVariables", arg0, arg1, arg2)
+ ret0, _ := ret[0].(string)
+ return ret0
+}
+
+// ExpandVariables indicates an expected call of ExpandVariables.
+func (mr *MockConfigServiceMockRecorder) ExpandVariables(arg0, arg1, arg2 interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExpandVariables", reflect.TypeOf((*MockConfigService)(nil).ExpandVariables), arg0, arg1, arg2)
+}
diff --git a/internal/manager/api_impl/test-flamenco-manager.yaml b/internal/manager/api_impl/test-flamenco-manager.yaml
new file mode 100644
index 00000000..65a596d6
--- /dev/null
+++ b/internal/manager/api_impl/test-flamenco-manager.yaml
@@ -0,0 +1,60 @@
+# This file is loaded by unit tests.
+_meta:
+ version: 3
+mode: develop
+listen: '[::0]:8083'
+own_url: http://192.168.3.108:8083/
+flamenco: http://localhost:51234/
+manager_id: 5852bc5198377351f95d103e
+manager_secret: SRVwA7wAxPRfudvqTDOLXwPn1cDRIlADz5Ef9kHk7d52Us
+
+task_logs_path: /tmp/flamenco-unittests
+blacklist_threshold: 3
+
+shaman:
+ enabled: false
+
+variables:
+ blender:
+ direction: oneway
+ values:
+ - audience: users
+ platform: linux
+ value: /linux/path/to/blender
+ - audience: workers
+ platform: linux
+ value: /opt/myblenderbuild/blender
+ - platform: windows
+ value: 'c:/temp/blender.exe'
+ - platform: darwin
+ value: /opt/myblenderbuild/blender
+ ffmpeg:
+ direction: oneway
+ values:
+ - platform: linux
+ value: /usr/bin/ffmpeg
+ - platform: windows
+ value: xxx
+ - platform: darwin
+ value: xxx
+ render_long:
+ direction: twoway
+ values:
+ - platform: windows
+ value: s:/flamenco/render/long
+ - platform: linux
+ value: /shared/flamenco/render/long
+ - platform: darwin
+ value: /Volume/shared/flamenco/render/long
+
+ job_storage:
+ direction: twoway
+ values:
+ - platform: windows
+ value: s:/flamenco/jobs
+ - platform: linux
+ value: /shared/flamenco/jobs
+ - platform: darwin
+ value: /Volume/shared/flamenco/jobs
+ - platform: autumn
+ value: hey
diff --git a/internal/manager/api_impl/test_support.go b/internal/manager/api_impl/test_support.go
index 9d2868b6..571c335d 100644
--- a/internal/manager/api_impl/test_support.go
+++ b/internal/manager/api_impl/test_support.go
@@ -39,18 +39,21 @@ type mockedFlamenco struct {
flamenco *Flamenco
jobCompiler *mocks.MockJobCompiler
persistence *mocks.MockPersistenceService
+ config *mocks.MockConfigService
}
func newMockedFlamenco(mockCtrl *gomock.Controller) mockedFlamenco {
jc := mocks.NewMockJobCompiler(mockCtrl)
ps := mocks.NewMockPersistenceService(mockCtrl)
ls := mocks.NewMockLogStorage(mockCtrl)
- f := NewFlamenco(jc, ps, ls)
+ cs := mocks.NewMockConfigService(mockCtrl)
+ f := NewFlamenco(jc, ps, ls, cs)
return mockedFlamenco{
flamenco: f,
jobCompiler: jc,
persistence: ps,
+ config: cs,
}
}
diff --git a/internal/manager/api_impl/varrepl.go b/internal/manager/api_impl/varrepl.go
new file mode 100644
index 00000000..bc5816fc
--- /dev/null
+++ b/internal/manager/api_impl/varrepl.go
@@ -0,0 +1,59 @@
+package api_impl
+
+/* ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * Original Code Copyright (C) 2022 Blender Foundation.
+ *
+ * This file is part of Flamenco.
+ *
+ * Flamenco is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later
+ * version.
+ *
+ * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * Flamenco. If not, see .
+ *
+ * ***** END GPL LICENSE BLOCK ***** */
+
+import (
+ "reflect"
+
+ "gitlab.com/blender/flamenco-ng-poc/internal/manager/persistence"
+)
+
+var stringType = reflect.TypeOf("somestring")
+
+type VariableReplacer interface {
+ ExpandVariables(valueToExpand, audience, platform string) string
+}
+
+// replaceTaskVariables performs variable replacement for worker tasks.
+func replaceTaskVariables(replacer VariableReplacer, task persistence.Task, worker persistence.Worker) persistence.Task {
+ repl := func(value string) string {
+ return replacer.ExpandVariables(value, "workers", worker.Platform)
+ }
+
+ for cmdIndex, cmd := range task.Commands {
+ for key, value := range cmd.Parameters {
+ switch v := value.(type) {
+ case string:
+ task.Commands[cmdIndex].Parameters[key] = repl(v)
+ case []string:
+ replaced := make([]string, len(v))
+ for idx := range v {
+ replaced[idx] = repl(v[idx])
+ }
+ task.Commands[cmdIndex].Parameters[key] = replaced
+ default:
+ continue
+ }
+ }
+ }
+
+ return task
+}
diff --git a/internal/manager/api_impl/varrepl_test.go b/internal/manager/api_impl/varrepl_test.go
new file mode 100644
index 00000000..7ec53513
--- /dev/null
+++ b/internal/manager/api_impl/varrepl_test.go
@@ -0,0 +1,113 @@
+package api_impl
+
+/* ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * Original Code Copyright (C) 2022 Blender Foundation.
+ *
+ * This file is part of Flamenco.
+ *
+ * Flamenco is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later
+ * version.
+ *
+ * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * Flamenco. If not, see .
+ *
+ * ***** END GPL LICENSE BLOCK ***** */
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "gitlab.com/blender/flamenco-ng-poc/internal/manager/config"
+ "gitlab.com/blender/flamenco-ng-poc/internal/manager/persistence"
+)
+
+func varreplTestTask() persistence.Task {
+ return persistence.Task{
+ Commands: []persistence.Command{
+ {Name: "echo", Parameters: persistence.StringInterfaceMap{
+ "message": "Running Blender from {blender} {blender}"}},
+ {Name: "sleep", Parameters: persistence.StringInterfaceMap{
+ "{blender}": 3}},
+ {
+ Name: "blender_render",
+ Parameters: persistence.StringInterfaceMap{
+ "filepath": "{job_storage}/sybren/2017-06-08-181223.625800-sybren-flamenco-test.flamenco/flamenco-test.flamenco.blend",
+ "exe": "{blender}",
+ "otherpath": "{hey}/haha",
+ "frames": "47",
+ "cycles_chunk": 1.0,
+ "args": []string{"--render-out", "{render_long}/sybren/blender-cloud-addon/flamenco-test__intermediate/render-smpl-0001-0084-frm-######"},
+ },
+ },
+ },
+ }
+}
+
+func TestReplaceVariables(t *testing.T) {
+ worker := persistence.Worker{Platform: "linux"}
+ task := varreplTestTask()
+ conf := config.GetTestConfig()
+ replacedTask := replaceTaskVariables(&conf, task, worker)
+
+ // Single string value.
+ assert.Equal(t,
+ "/opt/myblenderbuild/blender",
+ replacedTask.Commands[2].Parameters["exe"],
+ )
+
+ // Array value.
+ assert.Equal(t,
+ []string{"--render-out", "/shared/flamenco/render/long/sybren/blender-cloud-addon/flamenco-test__intermediate/render-smpl-0001-0084-frm-######"},
+ replacedTask.Commands[2].Parameters["args"],
+ )
+
+ // Substitution should happen as often as needed.
+ assert.Equal(t,
+ "Running Blender from /opt/myblenderbuild/blender /opt/myblenderbuild/blender",
+ replacedTask.Commands[0].Parameters["message"],
+ )
+
+ // No substitution should happen on keys, just on values.
+ assert.Equal(t, 3, replacedTask.Commands[1].Parameters["{blender}"])
+}
+
+func TestReplacePathsWindows(t *testing.T) {
+ worker := persistence.Worker{Platform: "windows"}
+ task := varreplTestTask()
+ conf := config.GetTestConfig()
+ replacedTask := replaceTaskVariables(&conf, task, worker)
+
+ assert.Equal(t,
+ "s:/flamenco/jobs/sybren/2017-06-08-181223.625800-sybren-flamenco-test.flamenco/flamenco-test.flamenco.blend",
+ replacedTask.Commands[2].Parameters["filepath"],
+ )
+ assert.Equal(t,
+ []string{"--render-out", "s:/flamenco/render/long/sybren/blender-cloud-addon/flamenco-test__intermediate/render-smpl-0001-0084-frm-######"},
+ replacedTask.Commands[2].Parameters["args"],
+ )
+ assert.Equal(t, "{hey}/haha", replacedTask.Commands[2].Parameters["otherpath"])
+}
+
+func TestReplacePathsUnknownOS(t *testing.T) {
+ worker := persistence.Worker{Platform: "autumn"}
+ task := varreplTestTask()
+ conf := config.GetTestConfig()
+ replacedTask := replaceTaskVariables(&conf, task, worker)
+
+ assert.Equal(t,
+ "hey/sybren/2017-06-08-181223.625800-sybren-flamenco-test.flamenco/flamenco-test.flamenco.blend",
+ replacedTask.Commands[2].Parameters["filepath"],
+ )
+ assert.Equal(t,
+ []string{"--render-out", "{render_long}/sybren/blender-cloud-addon/flamenco-test__intermediate/render-smpl-0001-0084-frm-######"},
+ replacedTask.Commands[2].Parameters["args"],
+ )
+ assert.Equal(t, "{hey}/haha", replacedTask.Commands[2].Parameters["otherpath"])
+}
diff --git a/internal/manager/config/service.go b/internal/manager/config/service.go
new file mode 100644
index 00000000..8a4f0d9b
--- /dev/null
+++ b/internal/manager/config/service.go
@@ -0,0 +1,47 @@
+package config
+
+/* ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * Original Code Copyright (C) 2022 Blender Foundation.
+ *
+ * This file is part of Flamenco.
+ *
+ * Flamenco is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later
+ * version.
+ *
+ * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * Flamenco. If not, see .
+ *
+ * ***** END GPL LICENSE BLOCK ***** */
+
+// Service provides access to Flamenco Manager configuration.
+type Service struct {
+ config Conf
+}
+
+func NewService() *Service {
+ return &Service{}
+}
+
+func (s *Service) Load() error {
+ config, err := getConf()
+ if err != nil {
+ return err
+ }
+ s.config = config
+ return nil
+}
+
+func (s *Service) ExpandVariables(valueToExpand, audience, platform string) string {
+ return s.config.ExpandVariables(valueToExpand, audience, platform)
+}
+
+func (s *Service) Get() *Conf {
+ return &s.config
+}
diff --git a/internal/manager/config/settings.go b/internal/manager/config/settings.go
new file mode 100644
index 00000000..9ebd8fcf
--- /dev/null
+++ b/internal/manager/config/settings.go
@@ -0,0 +1,650 @@
+package config
+
+/* ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * Original Code Copyright (C) 2022 Blender Foundation.
+ *
+ * This file is part of Flamenco.
+ *
+ * Flamenco is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later
+ * version.
+ *
+ * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * Flamenco. If not, see .
+ *
+ * ***** END GPL LICENSE BLOCK ***** */
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path"
+ "runtime"
+ "strings"
+ "time"
+
+ "github.com/rs/zerolog/log"
+ yaml "gopkg.in/yaml.v2"
+
+ "gitlab.com/blender/flamenco-ng-poc/pkg/api"
+)
+
+const (
+ configFilename = "flamenco-manager.yaml"
+
+ latestConfigVersion = 3
+
+ // // relative to the Flamenco Server Base URL:
+ // jwtPublicKeysRelativeURL = "api/flamenco/jwt/public-keys"
+
+ defaultShamanFilestorePath = "/shared/flamenco/file-store"
+ defaultJobStorage = "/shared/flamenco/jobs"
+)
+
+var (
+ // ErrMissingVariablePlatform is returned when a variable doesn't declare any valid platform for a certain value.
+ ErrMissingVariablePlatform = errors.New("variable's value is missing platform declaration")
+ // ErrBadDirection is returned when a direction doesn't match "oneway" or "twoway"
+ ErrBadDirection = errors.New("variable's direction is invalid")
+
+ // Valid values for the "mode" config variable.
+ validModes = map[string]bool{
+ "develop": true,
+ "production": true,
+ }
+
+ // Valid values for the "audience" tag of a ConfV2 variable.
+ validAudiences = map[string]bool{
+ "all": true,
+ "workers": true,
+ "users": true,
+ }
+
+ // The default configuration, use DefaultConfig() to obtain a copy.
+ defaultConfig = Conf{
+ Base: Base{
+ Meta: ConfMeta{Version: latestConfigVersion},
+
+ Mode: "production",
+ ManagerName: "Flamenco Manager",
+ Listen: ":8080",
+ ListenHTTPS: ":8433",
+ DatabaseDSN: "host=localhost user=flamenco password=flamenco dbname=flamenco TimeZone=Europe/Amsterdam",
+ TaskLogsPath: "./task-logs",
+ // DownloadTaskSleep: 10 * time.Minute,
+ // DownloadTaskRecheckThrottle: 10 * time.Second,
+ // TaskUpdatePushMaxInterval: 5 * time.Second,
+ // TaskUpdatePushMaxCount: 3000,
+ // CancelTaskFetchInterval: 10 * time.Second,
+ ActiveTaskTimeoutInterval: 10 * time.Minute,
+ ActiveWorkerTimeoutInterval: 1 * time.Minute,
+ // FlamencoStr: defaultServerURL,
+
+ // // Days are assumed to be 24 hours long. This is not exactly accurate, but should
+ // // be accurate enough for this type of cleanup.
+ // TaskCleanupMaxAge: 14 * 24 * time.Hour,
+ SSDPDiscovery: false, // Only enable after SSDP discovery has been improved (avoid finding printers).
+ SSDPDeviceUUID: "64ad4c21-6042-4378-9cdf-478f88b4f990", // UUID specific for Flamenco v3.
+
+ BlacklistThreshold: 3,
+ TaskFailAfterSoftFailCount: 3,
+
+ WorkerCleanupStatus: []string{string(api.WorkerStatusOffline)},
+
+ TestTasks: TestTasks{
+ BlenderRender: BlenderRenderConfig{
+ JobStorage: "{job_storage}/test-jobs",
+ RenderOutput: "{render}/test-renders",
+ },
+ },
+
+ Shaman: ShamanConfig{
+ Enabled: true,
+ FileStorePath: defaultShamanFilestorePath,
+ GarbageCollect: ShamanGarbageCollect{
+ Period: 24 * time.Hour,
+ MaxAge: 31 * 24 * time.Hour,
+ ExtraCheckoutDirs: []string{},
+ },
+ },
+
+ // JWT: jwtauth.Config{
+ // DownloadKeysInterval: 1 * time.Hour,
+ // },
+ },
+
+ Variables: map[string]Variable{
+ "blender": {
+ Direction: "oneway",
+ Values: VariableValues{
+ VariableValue{Platform: "linux", Value: "/linux/path/to/blender --factory-startup --background"},
+ VariableValue{Platform: "windows", Value: "C:/windows/path/to/blender.exe --factory-startup --background"},
+ VariableValue{Platform: "darwin", Value: "/Volumes/Applications/Blender/blender --factory-startup --background"},
+ },
+ },
+ "ffmpeg": {
+ Direction: "oneway",
+ Values: VariableValues{
+ VariableValue{Platform: "linux", Value: "/usr/bin/ffmpeg"},
+ VariableValue{Platform: "windows", Value: "C:/windows/path/to/ffmpeg.exe"},
+ VariableValue{Platform: "darwin", Value: "/Volumes/Applications/FFmpeg/ffmpeg"},
+ },
+ },
+ "job_storage": {
+ Direction: "twoway",
+ Values: VariableValues{
+ VariableValue{Platform: "linux", Value: "/shared/flamenco/jobs"},
+ VariableValue{Platform: "windows", Value: "S:/flamenco/jobs"},
+ VariableValue{Platform: "darwin", Value: "/Volumes/Shared/flamenco/jobs"},
+ },
+ },
+ "render": {
+ Direction: "twoway",
+ Values: VariableValues{
+ VariableValue{Platform: "linux", Value: "/shared/flamenco/render"},
+ VariableValue{Platform: "windows", Value: "S:/flamenco/render"},
+ VariableValue{Platform: "darwin", Value: "/Volumes/Shared/flamenco/render"},
+ },
+ },
+ },
+ }
+)
+
+// BlenderRenderConfig represents the configuration required for a test render.
+type BlenderRenderConfig struct {
+ JobStorage string `yaml:"job_storage"`
+ RenderOutput string `yaml:"render_output"`
+}
+
+// TestTasks represents the 'test_tasks' key in the Manager's configuration file.
+type TestTasks struct {
+ BlenderRender BlenderRenderConfig `yaml:"test_blender_render"`
+}
+
+// ConfMeta contains configuration file metadata.
+type ConfMeta struct {
+ // Version of the config file structure.
+ Version int `yaml:"version"`
+}
+
+// Base contains those settings that are shared by all configuration versions.
+type Base struct {
+ Meta ConfMeta `yaml:"_meta"`
+
+ Mode string `yaml:"mode"` // either "develop" or "production"
+ ManagerName string `yaml:"manager_name"`
+ DatabaseDSN string `yaml:"database_url"`
+ TaskLogsPath string `yaml:"task_logs_path"`
+ Listen string `yaml:"listen"`
+ ListenHTTPS string `yaml:"listen_https"`
+ OwnURL string `yaml:"own_url"` // sent to workers via SSDP/UPnP
+
+ // TLS certificate management. TLSxxx has priority over ACME.
+ TLSKey string `yaml:"tlskey"`
+ TLSCert string `yaml:"tlscert"`
+ ACMEDomainName string `yaml:"acme_domain_name"` // for the ACME Let's Encrypt client
+
+ ActiveTaskTimeoutInterval time.Duration `yaml:"active_task_timeout_interval"`
+ ActiveWorkerTimeoutInterval time.Duration `yaml:"active_worker_timeout_interval"`
+
+ WorkerCleanupMaxAge time.Duration `yaml:"worker_cleanup_max_age"`
+ WorkerCleanupStatus []string `yaml:"worker_cleanup_status"`
+
+ /* This many failures (on a given job+task type combination) will ban a worker
+ * from that task type on that job. */
+ BlacklistThreshold int `yaml:"blacklist_threshold"`
+
+ // When this many workers have tried the task and failed, it will be hard-failed
+ // (even when there are workers left that could technically retry the task).
+ TaskFailAfterSoftFailCount int `yaml:"task_fail_after_softfail_count"`
+
+ SSDPDiscovery bool `yaml:"ssdp_discovery"`
+ SSDPDeviceUUID string `yaml:"ssdp_device_uuid"`
+
+ TestTasks TestTasks `yaml:"test_tasks"`
+
+ // Shaman configuration settings.
+ Shaman ShamanConfig `yaml:"shaman"`
+
+ // Authentication settings.
+ // JWT jwtauth.Config `yaml:"user_authentication"`
+ WorkerRegistrationSecret string `yaml:"worker_registration_secret"`
+
+ // Dynamic worker pools (Azure Batch, Google Compute, AWS, that sort).
+ // DynamicPoolPlatforms *dppoller.Config `yaml:"dynamic_pool_platforms,omitempty"`
+
+ // Websetup *WebsetupConf `yaml:"websetup,omitempty"`
+}
+
+type ShamanConfig struct {
+ Enabled bool `yaml:"enabled"`
+ FileStorePath string `yaml:"fileStorePath"`
+ GarbageCollect ShamanGarbageCollect `yaml:"garbageCollect"`
+}
+
+// GarbageCollect contains the config options for the GC.
+type ShamanGarbageCollect struct {
+ // How frequently garbage collection is performed on the file store:
+ Period time.Duration `yaml:"period"`
+ // How old files must be before they are GC'd:
+ MaxAge time.Duration `yaml:"maxAge"`
+ // Paths to check for symlinks before GC'ing files.
+ ExtraCheckoutDirs []string `yaml:"extraCheckoutPaths"`
+
+ // Used by the -gc CLI arg to silently disable the garbage collector
+ // while we're performing a manual sweep.
+ SilentlyDisable bool `yaml:"-"`
+}
+
+// Conf is the latest version of the configuration.
+// Currently it is version 3.
+type Conf struct {
+ Base `yaml:",inline"`
+
+ // Variable name → Variable definition
+ Variables map[string]Variable `yaml:"variables"`
+
+ // audience + platform + variable name → variable value.
+ // Used to look up variables for a given platform and audience.
+ // The 'audience' is never "all" or ""; only concrete audiences are stored here.
+ VariablesLookup map[string]map[string]map[string]string `yaml:"-"`
+}
+
+// Variable defines a configuration variable.
+type Variable struct {
+ // Either "oneway" or "twoway"
+ Direction string `yaml:"direction" json:"direction"`
+ // Mapping from variable value to audience/platform definition.
+ Values VariableValues `yaml:"values" json:"values"`
+}
+
+// VariableValues is the list of values of a variable.
+type VariableValues []VariableValue
+
+// VariableValue defines which audience and platform see which value.
+type VariableValue struct {
+ // Audience defines who will use this variable, either "all", "workers", or "users". Empty string is "all".
+ Audience string `yaml:"audience,omitempty" json:"audience,omitempty"`
+
+ // Platforms that use this value. Only one of "Platform" and "Platforms" may be set.
+ Platform string `yaml:"platform,omitempty" json:"platform,omitempty"`
+ Platforms []string `yaml:"platforms,omitempty,flow" json:"platforms,omitempty,flow"`
+
+ // The actual value of the variable for this audience+platform.
+ Value string `yaml:"value" json:"value"`
+}
+
+// WebsetupConf are settings used by the web setup mode.
+// type WebsetupConf struct {
+// // When true, the websetup will hide certain settings that are infrastructure-specific.
+// // For example, it hides MongoDB choice, port numbers, task log directory, all kind of
+// // hosting-specific things. This is used, for example, by the automated Azure deployment
+// // to avoid messing up settings that are specific to that particular installation.
+// HideInfraSettings bool `yaml:"hide_infra_settings"`
+// }
+
+// getConf parses flamenco-manager.yaml and returns its contents as a Conf object.
+func getConf() (Conf, error) {
+ return loadConf(configFilename)
+}
+
+// DefaultConfig returns a copy of the default configuration.
+func DefaultConfig() Conf {
+ c := defaultConfig
+ c.Meta.Version = latestConfigVersion
+ c.constructVariableLookupTable()
+ return c
+}
+
+// loadConf parses the given file and returns its contents as a Conf object.
+func loadConf(filename string) (Conf, error) {
+ yamlFile, err := os.ReadFile(filename)
+ if err != nil {
+ return DefaultConfig(), err
+ }
+
+ // First parse attempt, find the version.
+ baseConf := Base{}
+ if err := yaml.Unmarshal(yamlFile, &baseConf); err != nil {
+ return Conf{}, fmt.Errorf("unable to parse %s: %w", filename, err)
+ }
+
+ // Versioning was supported from Flamenco config v1 to v2, but not further.
+ if baseConf.Meta.Version != latestConfigVersion {
+ return Conf{}, fmt.Errorf(
+ "configuration file %s version %d, but only version %d is supported",
+ filename, baseConf.Meta.Version, latestConfigVersion)
+ }
+
+ // Second parse attempt, based on the version found.
+ c := DefaultConfig()
+ if err := yaml.Unmarshal(yamlFile, &c); err != nil {
+ return c, fmt.Errorf("unable to parse %s: %w", filename, err)
+ }
+
+ c.constructVariableLookupTable()
+ c.parseURLs()
+ c.checkMode(c.Mode)
+ c.checkDatabase()
+ c.checkVariables()
+ c.checkTLS()
+
+ return c, nil
+}
+
+func (c *Conf) constructVariableLookupTable() {
+ lookup := map[string]map[string]map[string]string{}
+
+ // Construct a list of all audiences except "" and "all"
+ concreteAudiences := []string{}
+ isWildcard := map[string]bool{"": true, "all": true}
+ for audience := range validAudiences {
+ if isWildcard[audience] {
+ continue
+ }
+ concreteAudiences = append(concreteAudiences, audience)
+ }
+ log.Debug().
+ Strs("concreteAudiences", concreteAudiences).
+ Interface("isWildcard", isWildcard).
+ Msg("constructing variable lookup table")
+
+ // setValue expands wildcard audiences into concrete ones.
+ var setValue func(audience, platform, name, value string)
+ setValue = func(audience, platform, name, value string) {
+ if isWildcard[audience] {
+ for _, aud := range concreteAudiences {
+ setValue(aud, platform, name, value)
+ }
+ return
+ }
+
+ if lookup[audience] == nil {
+ lookup[audience] = map[string]map[string]string{}
+ }
+ if lookup[audience][platform] == nil {
+ lookup[audience][platform] = map[string]string{}
+ }
+ log.Debug().
+ Str("audience", audience).
+ Str("platform", platform).
+ Str("name", name).
+ Str("value", value).
+ Msg("setting variable")
+ lookup[audience][platform][name] = value
+ }
+
+ // Construct the lookup table for each audience+platform+name
+ for name, variable := range c.Variables {
+ log.Debug().
+ Str("name", name).
+ Interface("variable", variable).
+ Msg("handling variable")
+ for _, value := range variable.Values {
+
+ // Two-way values should not end in path separator.
+ // Given a variable 'apps' with value '/path/to/apps',
+ // '/path/to/apps/blender' should be remapped to '{apps}/blender'.
+ if variable.Direction == "twoway" {
+ if strings.Contains(value.Value, "\\") {
+ log.Warn().
+ Str("variable", name).
+ Str("audience", value.Audience).
+ Str("platform", value.Platform).
+ Str("value", value.Value).
+ Msg("Backslash found in variable value. Change paths to use forward slashes instead.")
+ }
+ value.Value = strings.TrimRight(value.Value, "/")
+ }
+
+ if value.Platform != "" {
+ setValue(value.Audience, value.Platform, name, value.Value)
+ }
+ for _, platform := range value.Platforms {
+ setValue(value.Audience, platform, name, value.Value)
+ }
+ }
+ }
+ log.Debug().
+ Interface("variables", c.Variables).
+ Interface("lookup", lookup).
+ Msg("constructed lookup table")
+ c.VariablesLookup = lookup
+}
+
+// ExpandVariables converts "{variable name}" to the value that belongs to the given audience and platform.
+func (c *Conf) ExpandVariables(valueToExpand, audience, platform string) string {
+ audienceMap := c.VariablesLookup[audience]
+ if audienceMap == nil {
+ log.Warn().
+ Str("valueToExpand", valueToExpand).
+ Str("audience", audience).
+ Str("platform", platform).
+ Msg("no variables defined for this audience")
+ return valueToExpand
+ }
+
+ platformMap := audienceMap[platform]
+ if platformMap == nil {
+ log.Warn().
+ Str("valueToExpand", valueToExpand).
+ Str("audience", audience).
+ Str("platform", platform).
+ Msg("no variables defined for this platform given this audience")
+ return valueToExpand
+ }
+
+ // Variable replacement
+ for varname, varvalue := range platformMap {
+ placeholder := fmt.Sprintf("{%s}", varname)
+ valueToExpand = strings.Replace(valueToExpand, placeholder, varvalue, -1)
+ }
+
+ return valueToExpand
+}
+
+// checkVariables performs some basic checks on variable definitions.
+// Note that the returned error only reflects the last-found error.
+// All errors are logged, though.
+func (c *Conf) checkVariables() error {
+ var err error
+
+ directionNames := []string{"oneway", "twoway"}
+ validDirections := map[string]bool{}
+ for _, direction := range directionNames {
+ validDirections[direction] = true
+ }
+
+ for name, variable := range c.Variables {
+ if !validDirections[variable.Direction] {
+ log.Error().
+ Str("name", name).
+ Str("direction", variable.Direction).
+ Strs("validChoices", directionNames).
+ Msg("variable has invalid direction")
+ err = ErrBadDirection
+ }
+ for valueIndex, value := range variable.Values {
+ // No platforms at all.
+ if value.Platform == "" && len(value.Platforms) == 0 {
+ log.Error().
+ Str("name", name).
+ Interface("value", value).
+ Msg("variable has a platformless value")
+ err = ErrMissingVariablePlatform
+ continue
+ }
+
+ // Both Platform and Platforms.
+ if value.Platform != "" && len(value.Platforms) > 0 {
+ log.Warn().
+ Str("name", name).
+ Interface("value", value).
+ Str("platform", value.Platform).
+ Strs("platforms", value.Platforms).
+ Msg("variable has a both 'platform' and 'platforms' set")
+ value.Platforms = append(value.Platforms, value.Platform)
+ value.Platform = ""
+ }
+
+ if value.Audience == "" {
+ value.Audience = "all"
+ } else if !validAudiences[value.Audience] {
+ log.Error().
+ Str("name", name).
+ Interface("value", value).
+ Str("audience", value.Audience).
+ Msg("variable invalid audience")
+ }
+
+ variable.Values[valueIndex] = value
+ }
+ }
+
+ return err
+}
+
+func (c *Conf) checkDatabase() {
+ c.DatabaseDSN = strings.TrimSpace(c.DatabaseDSN)
+}
+
+// Overwrite stores this configuration object as flamenco-manager.yaml.
+func (c *Conf) Overwrite() error {
+ tempFilename := configFilename + "~"
+ if err := c.Write(tempFilename); err != nil {
+ return fmt.Errorf("error writing config to %s: %w", tempFilename, err)
+ }
+ if err := os.Rename(tempFilename, configFilename); err != nil {
+ return fmt.Errorf("error moving %s to %s: %w", tempFilename, configFilename, err)
+ }
+
+ log.Info().Str("filename", configFilename).Msg("saved configuration to file")
+ return nil
+}
+
+// Write saves the current in-memory configuration to a YAML file.
+func (c *Conf) Write(filename string) error {
+ data, err := yaml.Marshal(c)
+ if err != nil {
+ return err
+ }
+
+ f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+ if err != nil {
+ return err
+ }
+ fmt.Fprintln(f, "# Configuration file for Flamenco Manager.")
+ fmt.Fprintln(f, "# For an explanation of the fields, refer to flamenco-manager-example.yaml")
+ fmt.Fprintln(f, "#")
+ fmt.Fprintln(f, "# NOTE: this file will be overwritten by Flamenco Manager's web-based configuration system.")
+ fmt.Fprintln(f, "#")
+ now := time.Now()
+ fmt.Fprintf(f, "# This file was written on %s\n\n", now.Format("2006-01-02 15:04:05 -07:00"))
+
+ n, err := f.Write(data)
+ if err != nil {
+ return err
+ }
+ if n < len(data) {
+ return io.ErrShortWrite
+ }
+ if err = f.Close(); err != nil {
+ return err
+ }
+
+ log.Debug().Str("filename", filename).Msg("config file written")
+ return nil
+}
+
+// HasCustomTLS returns true if both the TLS certificate and key files are configured.
+func (c *Conf) HasCustomTLS() bool {
+ return c.TLSCert != "" && c.TLSKey != ""
+}
+
+// HasTLS returns true if either a custom certificate or ACME/Let's Encrypt is used.
+func (c *Conf) HasTLS() bool {
+ return c.ACMEDomainName != "" || c.HasCustomTLS()
+}
+
+// OverrideMode checks the mode parameter for validity and logs that it's being overridden.
+func (c *Conf) OverrideMode(mode string) {
+ if mode == c.Mode {
+ log.Warn().Str("mode", mode).Msg("trying to override run mode with current value; ignoring")
+ return
+ }
+ c.checkMode(mode)
+ log.Warn().
+ Str("configured_mode", c.Mode).
+ Str("current_mode", mode).
+ Msg("overriding run mode")
+ c.Mode = mode
+}
+
+func (c *Conf) checkMode(mode string) {
+ // Check mode for validity
+ if !validModes[mode] {
+ keys := make([]string, 0, len(validModes))
+ for k := range validModes {
+ keys = append(keys, k)
+ }
+ log.Error().
+ Strs("valid_values", keys).
+ Str("current_value", mode).
+ Msg("bad value for 'mode' configuration parameter")
+ }
+}
+
+func (c *Conf) checkTLS() {
+ hasTLS := c.HasCustomTLS()
+
+ if hasTLS && c.ListenHTTPS == "" {
+ c.ListenHTTPS = c.Listen
+ c.Listen = ""
+ }
+
+ if !hasTLS || c.ACMEDomainName == "" {
+ return
+ }
+
+ log.Warn().
+ Str("tlscert", c.TLSCert).
+ Str("tlskey", c.TLSKey).
+ Str("acme_domain_name", c.ACMEDomainName).
+ Msg("ACME/Let's Encrypt will not be used because custom certificate is specified")
+ c.ACMEDomainName = ""
+}
+
+func (c *Conf) parseURLs() {
+ // var err error
+ // if jwtURL, err := c.Flamenco.Parse(jwtPublicKeysRelativeURL); err != nil {
+ // log.WithFields(log.Fields{
+ // "url": c.Flamenco.String(),
+ // log.ErrorKey: err,
+ // }).Error("unable to construct URL to get JWT public keys")
+ // } else {
+ // c.JWT.PublicKeysURL = jwtURL.String()
+ // }
+}
+
+// GetTestConfig returns the configuration for unit tests.
+// The config is loaded from `test-flamenco-manager.yaml` in the directory
+// containing the caller's source.
+func GetTestConfig() Conf {
+ _, myFilename, _, _ := runtime.Caller(1)
+ myDir := path.Dir(myFilename)
+
+ filepath := path.Join(myDir, "test-flamenco-manager.yaml")
+ conf, err := loadConf(filepath)
+ if err != nil {
+ log.Fatal().Err(err).Str("file", filepath).Msg("unable to load test config")
+ }
+
+ return conf
+}
diff --git a/internal/manager/config/settings_test.go b/internal/manager/config/settings_test.go
new file mode 100644
index 00000000..cb185acb
--- /dev/null
+++ b/internal/manager/config/settings_test.go
@@ -0,0 +1,68 @@
+package config
+
+/* ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * Original Code Copyright (C) 2022 Blender Foundation.
+ *
+ * This file is part of Flamenco.
+ *
+ * Flamenco is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later
+ * version.
+ *
+ * Flamenco is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * Flamenco. If not, see .
+ *
+ * ***** END GPL LICENSE BLOCK ***** */
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestDefaultSettings(t *testing.T) {
+ config, err := loadConf("nonexistant.yaml")
+ assert.NotNil(t, err) // should indicate an error to open the file.
+
+ // The settings should contain the defaults, though.
+ assert.Equal(t, latestConfigVersion, config.Meta.Version)
+ assert.Equal(t, "./task-logs", config.TaskLogsPath)
+ assert.Equal(t, "64ad4c21-6042-4378-9cdf-478f88b4f990", config.SSDPDeviceUUID)
+
+ assert.Contains(t, config.Variables, "job_storage")
+ assert.Contains(t, config.Variables, "render")
+ assert.Equal(t, "oneway", config.Variables["ffmpeg"].Direction)
+ assert.Equal(t, "/usr/bin/ffmpeg", config.Variables["ffmpeg"].Values[0].Value)
+ assert.Equal(t, "linux", config.Variables["ffmpeg"].Values[0].Platform)
+
+ linuxPVars, ok := config.VariablesLookup["workers"]["linux"]
+ assert.True(t, ok, "workers/linux should have variables: %v", config.VariablesLookup)
+ assert.Equal(t, "/shared/flamenco/jobs", linuxPVars["job_storage"])
+
+ winPVars, ok := config.VariablesLookup["users"]["windows"]
+ assert.True(t, ok)
+ assert.Equal(t, "S:/flamenco/jobs", winPVars["job_storage"])
+}
+
+func TestVariableValidation(t *testing.T) {
+ c := DefaultConfig()
+
+ platformless := c.Variables["blender"]
+ platformless.Values = VariableValues{
+ VariableValue{Value: "/path/to/blender"},
+ VariableValue{Platform: "linux", Value: "/valid/path/blender"},
+ }
+ c.Variables["blender"] = platformless
+
+ err := c.checkVariables()
+ assert.Equal(t, ErrMissingVariablePlatform, err)
+
+ assert.Equal(t, c.Variables["blender"].Values[0].Value, "/path/to/blender")
+ assert.Equal(t, c.Variables["blender"].Values[1].Value, "/valid/path/blender")
+}