Simpler, Better, Faster, Stronger

Distributed rendering with Flamenco v3

dr. Sybren | Blender Conference | 2022-10-27

Introduction

dr. Sybren
software dev @ Blender HQ

keep questions until the end

Overview

• What is a render farm?
• What changed since Flamenco v2?
• 5-minute install
• Storage / Shaman
• Job Types
• Get Involved!

What is a

Render Farm

?

What Changed?

BConf 2019

60 minute workshop

barely got things working

Design Principles

1. Simplicity
2. Transparency
3. Interactivity
4. Robustness

Target Audience

• Small studios & home users
• 1-10 artists
• 1-100 computers
• NFS/SMB/CIFS file sharing
• All on one LAN
• Linux is main platform, macOS & Windows "also supported"

Install in 5 minutes

Mandatory fallback video: https://youtu.be/O728EFaXuBk

7 Steps to Profit

1. Download & Extract
2. Pick Storage
3. Install Blender
4. Start Manager
5. Install add-on
6. Submit a file
7. Start the Worker
8. Profit!

Storage

• Simple
• Efficient
• Isolated

Pick two

Storage

Shaman

• Identify files by their SHA256 checksum
• Only upload new files
• Creates directories, symlinks files

Variables

Variables

Two-Way Variables

for mixed-platform farms

Two-Way Variables

Two-Way Variables

Job Types

scripts/echo_sleep_test.js

JavaScript with a hint of Python

const JOB_TYPE = {
  label: "Echo Sleep Test",
  settings: [
      { key: "message", type: "string", required: true },
      { key: "sleep_duration_seconds", type: "int32", default: 1 },
      { key: "sleep_repeats", type: "int32", default: 1 },
  ]
};


function compileJob(job) {
  const settings = job.settings;

  const echoTask = author.Task("echo", "misc");
  echoTask.addCommand(author.Command("echo", {message: settings.message}));
  job.addTask(echoTask);

  for (let repeat=0; repeat < settings.sleep_repeats; repeat++) {
    const sleepTask = author.Task("sleep", "misc")
    sleepTask.addCommand(author.Command("sleep", {duration_in_seconds: settings.sleep_duration_seconds}))
    sleepTask.addDependency(echoTask); // Ensure sleeping happens after echo, and not at the same time.
    job.addTask(sleepTask);
  }
}

scripts/echo_sleep_test.js

            const JOB_TYPE = {
                label: "Simple Blender Render",
                settings: [
                    // Settings for artists to determine:
                    { key: "frames", type: "string", required: true, eval: "f'{C.scene.frame_start}-{C.scene.frame_end}'",
                      description: "Frame range to render. Examples: '47', '1-30', '3, 5-10, 47-327'" },
                    { key: "chunk_size", type: "int32", default: 1, description: "Number of frames to render in one Blender render task",
                      visible: "submission" },

                    // render_output_root + add_path_components determine the value of render_output_path.
                    { key: "render_output_root", type: "string", subtype: "dir_path", required: true, visible: "submission",
                      description: "Base directory of where render output is stored. Will have some job-specific parts appended to it"},
                    { key: "add_path_components", type: "int32", required: true, default: 0, propargs: {min: 0, max: 32}, visible: "submission",
                      description: "Number of path components of the current blend file to use in the render output path"},
                    { key: "render_output_path", type: "string", subtype: "file_path", editable: false,
                      eval: "str(Path(abspath(settings.render_output_root), last_n_dir_parts(settings.add_path_components),
                             jobname, '{timestamp}', '######'))",
                      description: "Final file path of where render output will be saved"},

                    // Automatically evaluated settings:
                    { key: "blendfile", type: "string", required: true, description: "Path of the Blend file to render", visible: "web" },
                    { key: "fps", type: "float", eval: "C.scene.render.fps / C.scene.render.fps_base", visible: "hidden" },
                    { key: "format", type: "string", required: true, eval: "C.scene.render.image_settings.file_format", visible: "web" },
                    { key: "image_file_extension", type: "string", required: true, eval: "C.scene.render.file_extension", visible: "hidden",
                      description: "File extension used when rendering images" },
                ]
            };


            // Set of scene.render.image_settings.file_format values that produce
            // files which FFmpeg is known not to handle as input.
            const ffmpegIncompatibleImageFormats = new Set([
                "EXR",
                "MULTILAYER", // Old CLI-style format indicators
                "OPEN_EXR",
                "OPEN_EXR_MULTILAYER", // DNA values for these formats.
            ]);

            // File formats that would cause rendering to video.
            // This is not supported by this job type.
            const videoFormats = ['FFMPEG', 'AVI_RAW', 'AVI_JPEG'];

            function compileJob(job) {
                print("Blender Render job submitted");
                print("job: ", job);

                const settings = job.settings;
                if (videoFormats.indexOf(settings.format) >= 0) {
                    throw `This job type only renders images, and not "${settings.format}"`;
                }

                const renderOutput = renderOutputPath(job);

                // Make sure that when the job is investigated later, it shows the
                // actually-used render output:
                settings.render_output_path = renderOutput;

                const renderDir = path.dirname(renderOutput);
                const renderTasks = authorRenderTasks(settings, renderDir, renderOutput);
                const videoTask = authorCreateVideoTask(settings, renderDir);

                for (const rt of renderTasks) {
                    job.addTask(rt);
                }
                if (videoTask) {
                    // If there is a video task, all other tasks have to be done first.
                    for (const rt of renderTasks) {
                        videoTask.addDependency(rt);
                    }
                    job.addTask(videoTask);
                }
            }

            // Do field replacement on the render output path.
            function renderOutputPath(job) {
                let path = job.settings.render_output_path;
                if (!path) {
                    throw "no render_output_path setting!";
                }
                return path.replace(/{([^}]+)}/g, (match, group0) => {
                    switch (group0) {
                    case "timestamp":
                        return formatTimestampLocal(job.created);
                    default:
                        return match;
                    }
                });
            }

            function authorRenderTasks(settings, renderDir, renderOutput) {
                print("authorRenderTasks(", renderDir, renderOutput, ")");
                let renderTasks = [];
                let chunks = frameChunker(settings.frames, settings.chunk_size);
                for (let chunk of chunks) {
                    const task = author.Task(`render-${chunk}`, "blender");
                    const command = author.Command("blender-render", {
                        exe: "{blender}",
                        exeArgs: "{blenderArgs}",
                        argsBefore: [],
                        blendfile: settings.blendfile,
                        args: [
                            "--render-output", path.join(renderDir, path.basename(renderOutput)),
                            "--render-format", settings.format,
                            "--render-frame", chunk.replace("-", ".."), // Convert to Blender frame range notation.
                        ]
                    });
                    task.addCommand(command);
                    renderTasks.push(task);
                }
                return renderTasks;
            }

            function authorCreateVideoTask(settings, renderDir) {
                if (ffmpegIncompatibleImageFormats.has(settings.format)) {
                    print("Not authoring video task, FFmpeg-incompatible render output")
                    return;
                }
                if (!settings.fps) {
                    print("Not authoring video task, no FPS known:", settings);
                    return;
                }

                const stem = path.stem(settings.blendfile).replace('.flamenco', '');
                const outfile = path.join(renderDir, `${stem}-${settings.frames}.mp4`);
                const outfileExt = settings.image_file_extension;

                const task = author.Task('preview-video', 'ffmpeg');
                const command = author.Command("frames-to-video", {
                    exe: "ffmpeg",
                    fps: settings.fps,
                    inputGlob: path.join(renderDir, `*${outfileExt}`),
                    outputFile: outfile,
                    args: [
                        "-c:v", "h264",
                        "-crf", "20",
                        "-g", "18",
                        "-vf", "pad=ceil(iw/2)*2:ceil(ih/2)*2",
                        "-pix_fmt", "yuv420p",
                        "-r", settings.fps,
                        "-y", // Be sure to always pass either "-n" or "-y".
                    ],
                });
                task.addCommand(command);

                print(`Creating output video for ${settings.format}`);
                return task;
            }
            

simple_blender_render.js

Not Presented

• Worker Sleep Schedule
• Blocklisting & auto-requeueing for failing workers
• OpenAPI + the API explorer

Overview

• What is a render farm?
• What changed since Flamenco v2?
• 5-minute install
• Storage / Shaman
• Job Types
• Get Involved!

Get Involved!

flamenco.blender.org