From b441f3f3deeaec751a959795d089060c6e9bcba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sybren=20A=2E=20St=C3=BCvel?= Date: Tue, 21 Jun 2022 17:35:59 +0200 Subject: [PATCH] Manager: load job compiler scripts from disk as well If there is a `scripts` directory next to the current executable, load scripts from that directory as well. It is still required to restart the Manager in order to pick up changes to those scripts (including new/removed files), PLUS a refresh in the add-on. --- FEATURES.md | 2 +- internal/manager/job_compilers/file_loader.go | 153 ++++++++++++++++++ .../manager/job_compilers/job_compilers.go | 11 +- internal/manager/job_compilers/scripts.go | 46 ++++-- .../manager/job_compilers/scripts_embedded.go | 28 ---- .../manager/job_compilers/scripts_test.go | 7 +- 6 files changed, 201 insertions(+), 46 deletions(-) create mode 100644 internal/manager/job_compilers/file_loader.go delete mode 100644 internal/manager/job_compilers/scripts_embedded.go diff --git a/FEATURES.md b/FEATURES.md index 413fdac3..4ee8f463 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -61,7 +61,7 @@ Note that list is **not** in any specific order. - [x] Let Manager write to task log when it's assigned to a worker. - [ ] Worker sleep schedule -- [ ] Loading of job compiler scripts from disk +- [x] Loading of job compiler scripts from disk - [ ] CLI option to write built-in job compiler scripts to disk - [ ] Per-job last rendered image - [ ] Support pausing jobs. diff --git a/internal/manager/job_compilers/file_loader.go b/internal/manager/job_compilers/file_loader.go new file mode 100644 index 00000000..0fd6ad60 --- /dev/null +++ b/internal/manager/job_compilers/file_loader.go @@ -0,0 +1,153 @@ +package job_compilers + +// SPDX-License-Identifier: GPL-3.0-or-later + +import ( + "embed" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + + "github.com/rs/zerolog/log" +) + +var ( + // embeddedScriptsFS gives access to the embedded scripts. + embeddedScriptsFS fs.FS + + // onDiskScriptsFS gives access to the on-disk scripts, located in a `scripts` + // directory next to the `flamenco-manager` executable. + onDiskScriptsFS fs.FS = nil + + fileLoaderInitialised = false +) + +const scriptsDirName = "scripts" + +// Scripts from the `./scripts` subdirectory are embedded into the executable +// here. Note that accessing these files still requires explicit use of the +// `scripts/` subdirectory, which is abstracted away by `embeddedScriptFS`. +// +//go:embed scripts +var _embeddedScriptsFS embed.FS + +func initFileLoader() { + if fileLoaderInitialised { + return + } + + initEmbeddedFS() + initOnDiskFS() + + fileLoaderInitialised = true +} + +// getAvailableFilesystems returns the filesystems to load scripts from, where +// earlier ones have priority over later ones. +func getAvailableFilesystems() []fs.FS { + filesystems := []fs.FS{} + + if onDiskScriptsFS != nil { + filesystems = append(filesystems, onDiskScriptsFS) + } + + filesystems = append(filesystems, embeddedScriptsFS) + return filesystems +} + +// loadFileFromAnyFS iterates over the available filesystems to find the +// identified file, and returns its contents when found. +// +// Returns `os.ErrNotExist` if there is no filesystem that has this file. +func loadFileFromAnyFS(path string) ([]byte, error) { + filesystems := getAvailableFilesystems() + + for _, fs := range filesystems { + file, err := fs.Open(path) + if os.IsNotExist(err) { + continue + } + if err != nil { + return nil, fmt.Errorf("failed to open file %s on filesystem %s: %w", path, fs, err) + } + return io.ReadAll(file) + } + + return nil, os.ErrNotExist +} + +func initEmbeddedFS() { + // Find embedded filesystem. Unless there were issues with the build of + // Flamenco Manager, this should always be here. + var err error + embeddedScriptsFS, err = fs.Sub(_embeddedScriptsFS, "scripts") + if err != nil { + panic(fmt.Sprintf("failed to find embedded 'scripts' directory: %v", err)) + } +} + +func initOnDiskFS() { + exename, err := os.Executable() + if err != nil { + log.Error().Err(err).Msg("job compiler: unable to determine the path of the currently running executable") + return + } + logger := log.With().Str("executable", exename).Logger() + logger.Debug().Msg("job compiler: searching for scripts directory next to executable") + + // Try to find the scripts next to the executable. + scriptsDir, found := findOnDiskScriptsNextTo(exename) + if found { + log.Debug().Str("scriptsDir", scriptsDir).Msg("job compiler: found scripts directory next to executable") + onDiskScriptsFS = os.DirFS(scriptsDir) + return + } + + // Evaluate any symlinks and see if that produces a different path to the + // executable. + evalLinkExe, err := filepath.EvalSymlinks(exename) + if err != nil { + logger.Error().Err(err).Msg("job compiler: unable to evaluate any symlinks to the running executable") + return + } + if evalLinkExe == exename { + // Evaluating any symlinks didn't produce a different path; no need to do the same search twice. + return + } + + scriptsDir, found = findOnDiskScriptsNextTo(evalLinkExe) + if !found { + logger.Debug().Msg("job compiler: did not find scripts directory next to executable") + return + } + + log.Debug().Str("scriptsDir", scriptsDir).Msg("job compiler: found scripts directory next to executable") + onDiskScriptsFS = os.DirFS(scriptsDir) +} + +// Find the `scripts` directory sitting next to the currently-running executable. +// Return the directory path, and a 'found' boolean indicating whether that path +// is actually a directory. +func findOnDiskScriptsNextTo(exename string) (string, bool) { + scriptsDir := filepath.Join(filepath.Dir(exename), scriptsDirName) + + logger := log.With().Str("scriptsDir", scriptsDir).Logger() + logger.Trace().Msg("job compiler: finding on-disk scripts") + + stat, err := os.Stat(scriptsDir) + if os.IsNotExist(err) { + return scriptsDir, false + } + if err != nil { + logger.Warn().Err(err).Msg("job compiler: error accessing scripts directory") + return scriptsDir, false + } + if !stat.IsDir() { + logger.Debug().Msg("job compiler: ignoring 'scripts' next to executable; it is not a directory") + return scriptsDir, false + } + + return scriptsDir, true +} diff --git a/internal/manager/job_compilers/job_compilers.go b/internal/manager/job_compilers/job_compilers.go index 510f51b3..5ba76b5e 100644 --- a/internal/manager/job_compilers/job_compilers.go +++ b/internal/manager/job_compilers/job_compilers.go @@ -7,6 +7,7 @@ package job_compilers import ( "context" "errors" + "os" "sort" "sync" "time" @@ -53,6 +54,8 @@ type TimeService interface { // Load returns a job compiler service with all JS files loaded. func Load(ts TimeService) (*Service, error) { + initFileLoader() + service := Service{ compilers: map[string]Compiler{}, timeService: ts, @@ -64,16 +67,14 @@ func Load(ts TimeService) (*Service, error) { } staticFileLoader := func(path string) ([]byte, error) { - // TODO: this should try different filesystems, once we allow loading from - // disk as well. - content, err := loadScriptBytes(getEmbeddedScriptFS(), path) - if err != nil { + content, err := loadFileFromAnyFS(path) + if err == os.ErrNotExist { // The 'require' module uses this to try different variations of the path // in order to find it (without .js, with .js, etc.), so don't log any of // such errors. return nil, require.ModuleFileDoesNotExistError } - return content, nil + return content, err } service.registry = require.NewRegistry(require.WithLoader(staticFileLoader)) diff --git a/internal/manager/job_compilers/scripts.go b/internal/manager/job_compilers/scripts.go index eba75129..20982d7a 100644 --- a/internal/manager/job_compilers/scripts.go +++ b/internal/manager/job_compilers/scripts.go @@ -17,9 +17,33 @@ import ( // loadScripts iterates over all JavaScript files, compiles them, and stores the // result into `s.compilers`. func (s *Service) loadScripts() error { - compilers, err := loadScriptsFrom(getEmbeddedScriptFS()) - if err != nil { - return err + compilers := map[string]Compiler{} + + // Collect all job compilers. + for _, fs := range getAvailableFilesystems() { + compilersfromFS, err := loadScriptsFrom(fs) + if err != nil { + log.Error().Err(err).Interface("fs", fs).Msg("job compiler: error loading scripts") + continue + } + if len(compilersfromFS) == 0 { + continue + } + + log.Debug().Interface("fs", fs). + Int("numScripts", len(compilersfromFS)). + Msg("job compiler: found job compiler scripts") + + // Merge the returned compilers into the big map, skipping ones that were + // already there. + for name := range compilersfromFS { + _, found := compilers[name] + if found { + continue + } + + compilers[name] = compilersfromFS[name] + } } // Assign the new set of compilers in a thread-safe way. @@ -30,8 +54,8 @@ func (s *Service) loadScripts() error { return nil } -// loadScriptsFrom iterates over all given directory entries, compiles the -// files, and stores the result into `s.compilers`. +// loadScriptsFrom iterates over files in the root of the given filesystem, +// compiles the files, and returns the "name -> compiler" mapping. func loadScriptsFrom(filesystem fs.FS) (map[string]Compiler, error) { dirEntries, err := fs.ReadDir(filesystem, ".") if err != nil { @@ -41,12 +65,16 @@ func loadScriptsFrom(filesystem fs.FS) (map[string]Compiler, error) { compilers := map[string]Compiler{} for _, dirEntry := range dirEntries { + if !dirEntry.Type().IsRegular() { + continue + } + filename := dirEntry.Name() if !strings.HasSuffix(filename, ".js") { continue } - script_bytes, err := loadScriptBytes(filesystem, filename) + script_bytes, err := loadFileFromFS(filesystem, filename) if err != nil { log.Error().Err(err).Str("filename", filename).Msg("failed to read script") continue @@ -76,16 +104,16 @@ func loadScriptsFrom(filesystem fs.FS) (map[string]Compiler, error) { log.Debug(). Str("script", filename). Str("jobType", jobTypeName). - Msg("loaded script") + Msg("job compiler: loaded script") } return compilers, nil } -func loadScriptBytes(filesystem fs.FS, path string) ([]byte, error) { +func loadFileFromFS(filesystem fs.FS, path string) ([]byte, error) { file, err := filesystem.Open(path) if err != nil { - return nil, fmt.Errorf("failed to open embedded script: %w", err) + return nil, fmt.Errorf("failed to open file %s on filesystem %s: %w", path, filesystem, err) } return io.ReadAll(file) } diff --git a/internal/manager/job_compilers/scripts_embedded.go b/internal/manager/job_compilers/scripts_embedded.go deleted file mode 100644 index d8d1d8cc..00000000 --- a/internal/manager/job_compilers/scripts_embedded.go +++ /dev/null @@ -1,28 +0,0 @@ -package job_compilers - -// SPDX-License-Identifier: GPL-3.0-or-later - -import ( - "embed" - "fmt" - "io/fs" -) - -// Scripts from the `./scripts` subdirectory are embedded into the executable -// here. Note that accessing these files still requires explicit use of the -// `scripts/` subdirectory, which is abstracted away by `getEmbeddedScriptFS()`. -// It is recommended to use that function to get the embedded scripts -// filesystem. - -//go:embed scripts -var _embeddedScriptsFS embed.FS - -// getEmbeddedScriptFS returns the `fs.FS` interface that allows access to the -// embedded job compiler scripts. -func getEmbeddedScriptFS() fs.FS { - scriptsSubFS, err := fs.Sub(_embeddedScriptsFS, "scripts") - if err != nil { - panic(fmt.Sprintf("failed to find embedded 'scripts' directory: %v", err)) - } - return scriptsSubFS -} diff --git a/internal/manager/job_compilers/scripts_test.go b/internal/manager/job_compilers/scripts_test.go index a51c50d2..aee41e76 100644 --- a/internal/manager/job_compilers/scripts_test.go +++ b/internal/manager/job_compilers/scripts_test.go @@ -31,7 +31,8 @@ func TestLoadScriptsFrom_on_disk_js(t *testing.T) { } func TestLoadScriptsFrom_embedded(t *testing.T) { - compilers, err := loadScriptsFrom(getEmbeddedScriptFS()) + initEmbeddedFS() + compilers, err := loadScriptsFrom(embeddedScriptsFS) assert.NoError(t, err) expectKeys := map[string]bool{ @@ -43,10 +44,10 @@ func TestLoadScriptsFrom_embedded(t *testing.T) { func BenchmarkLoadScripts_fromEmbedded(b *testing.B) { zerolog.SetGlobalLevel(zerolog.Disabled) + initEmbeddedFS() - embeddedFS := getEmbeddedScriptFS() for i := 0; i < b.N; i++ { - compilers, err := loadScriptsFrom(embeddedFS) + compilers, err := loadScriptsFrom(embeddedScriptsFS) assert.NoError(b, err) assert.NotEmpty(b, compilers) }