From 0dc1a58b241217899c88945ea8f6f8dc8f39470e Mon Sep 17 00:00:00 2001 From: X-Ryl669 Date: Tue, 14 Dec 2021 07:22:06 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Configurations=20embed=20and=20retr?= =?UTF-8?q?ieve=20(#21321,=20#23303)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 + Marlin/Configuration_adv.h | 8 + Marlin/src/gcode/eeprom/M500-M504.cpp | 18 ++ Marlin/src/gcode/host/M115.cpp | 28 ++- Marlin/src/inc/Conditionals_adv.h | 6 + Marlin/src/inc/Warnings.cpp | 4 + .../pins/stm32g0/pins_BTT_SKR_MINI_E3_V3_0.h | 2 +- buildroot/bin/restore_configs | 2 +- .../scripts/SAMD51_grandcentral_m4.py | 1 - .../share/PlatformIO/scripts/__init__.py | 0 .../PlatformIO/scripts/common-dependencies.py | 87 ++------- .../share/PlatformIO/scripts/mc-apply.py | 69 +++++++ .../share/PlatformIO/scripts/preprocessor.py | 99 ++++++++++ .../share/PlatformIO/scripts/signature.py | 176 ++++++++++++++++++ docs/ConfigEmbedding.md | 19 ++ 15 files changed, 436 insertions(+), 86 deletions(-) create mode 100644 buildroot/share/PlatformIO/scripts/__init__.py create mode 100755 buildroot/share/PlatformIO/scripts/mc-apply.py create mode 100644 buildroot/share/PlatformIO/scripts/preprocessor.py create mode 100644 buildroot/share/PlatformIO/scripts/signature.py create mode 100644 docs/ConfigEmbedding.md diff --git a/.gitignore b/.gitignore index 39468dd60c..72fd117bd2 100755 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ # Generated files _Version.h bdf2u8g +marlin_config.json +mczip.h +*.gen *.sublime-workspace # diff --git a/Marlin/Configuration_adv.h b/Marlin/Configuration_adv.h index 6168cd786a..1878a6f258 100644 --- a/Marlin/Configuration_adv.h +++ b/Marlin/Configuration_adv.h @@ -1593,6 +1593,14 @@ #define SD_FIRMWARE_UPDATE_INACTIVE_VALUE 0xFF #endif + /** + * Enable this option if you have more than ~3K of unused flash space. + * Marlin will embed all settings in the firmware binary as compressed data. + * Use 'M503 C' to write the settings out to the SD Card as 'mc.zip'. + * See docs/ConfigEmbedding.md for details on how to use 'mc-apply.py'. + */ + //#define CONFIGURATION_EMBEDDING + // Add an optimized binary file transfer mode, initiated with 'M28 B1' //#define BINARY_FILE_TRANSFER diff --git a/Marlin/src/gcode/eeprom/M500-M504.cpp b/Marlin/src/gcode/eeprom/M500-M504.cpp index a06e98ad1e..a1f295ebde 100644 --- a/Marlin/src/gcode/eeprom/M500-M504.cpp +++ b/Marlin/src/gcode/eeprom/M500-M504.cpp @@ -25,6 +25,11 @@ #include "../../core/serial.h" #include "../../inc/MarlinConfig.h" +#if ENABLED(CONFIGURATION_EMBEDDING) + #include "../../sd/SdBaseFile.h" + #include "../../mczip.h" +#endif + /** * M500: Store settings in EEPROM */ @@ -50,9 +55,22 @@ void GcodeSuite::M502() { /** * M503: print settings currently in memory + * + * With CONFIGURATION_EMBEDDING: + * C : Save the full Marlin configuration to SD Card as "mc.zip" */ void GcodeSuite::M503() { (void)settings.report(!parser.boolval('S', true)); + + #if ENABLED(CONFIGURATION_EMBEDDING) + if (parser.seen_test('C')) { + SdBaseFile file; + const uint16_t size = sizeof(mc_zip); + // Need to create the config size on the SD card + if (file.open("mc.zip", O_WRITE|O_CREAT) && file.write(pgm_read_ptr(mc_zip), size) != -1 && file.close()) + SERIAL_ECHO_MSG("Configuration saved as 'mc.zip'"); + } + #endif } #endif // !DISABLE_M503 diff --git a/Marlin/src/gcode/host/M115.cpp b/Marlin/src/gcode/host/M115.cpp index 9a90acbf0a..08943ed5f2 100644 --- a/Marlin/src/gcode/host/M115.cpp +++ b/Marlin/src/gcode/host/M115.cpp @@ -24,7 +24,6 @@ #include "../../inc/MarlinConfig.h" #include "../queue.h" // for getting the command port - #if ENABLED(M115_GEOMETRY_REPORT) #include "../../module/motion.h" #endif @@ -33,13 +32,25 @@ #include "../../feature/caselight.h" #endif +//#define MINIMAL_CAP_LINES // Don't even mention the disabled capabilities + #if ENABLED(EXTENDED_CAPABILITIES_REPORT) - static void cap_line(FSTR_P const name, bool ena=false) { - SERIAL_ECHOPGM("Cap:"); - SERIAL_ECHOF(name); - SERIAL_CHAR(':', '0' + ena); - SERIAL_EOL(); - } + #if ENABLED(MINIMAL_CAP_LINES) + #define cap_line(S,C) if (C) _cap_line(S) + static void _cap_line(FSTR_P const name) { + SERIAL_ECHOPGM("Cap:"); + SERIAL_ECHOF(name); + SERIAL_ECHOLNPGM(":1"); + } + #else + #define cap_line(V...) _cap_line(V) + static void _cap_line(FSTR_P const name, bool ena=false) { + SERIAL_ECHOPGM("Cap:"); + SERIAL_ECHOF(name); + SERIAL_CHAR(':', '0' + ena); + SERIAL_EOL(); + } + #endif #endif /** @@ -167,6 +178,9 @@ void GcodeSuite::M115() { // MEATPACK Compression cap_line(F("MEATPACK"), SERIAL_IMPL.has_feature(port, SerialFeature::MeatPack)); + // CONFIG_EXPORT + cap_line(F("CONFIG_EXPORT"), ENABLED(CONFIG_EMBED_AND_SAVE_TO_SD)); + // Machine Geometry #if ENABLED(M115_GEOMETRY_REPORT) const xyz_pos_t bmin = { 0, 0, 0 }, diff --git a/Marlin/src/inc/Conditionals_adv.h b/Marlin/src/inc/Conditionals_adv.h index 22f1591d68..9864b8715c 100644 --- a/Marlin/src/inc/Conditionals_adv.h +++ b/Marlin/src/inc/Conditionals_adv.h @@ -1004,3 +1004,9 @@ #if EITHER(MEATPACK_ON_SERIAL_PORT_1, MEATPACK_ON_SERIAL_PORT_2) #define HAS_MEATPACK 1 #endif + +// AVR are (usually) too limited in resources to store the configuration into the binary +#if !defined(FORCE_CONFIG_EMBED) && (defined(__AVR__) || DISABLED(SDSUPPORT) || EITHER(SDCARD_READONLY, DISABLE_M503)) + #undef CONFIGURATION_EMBEDDING + #define CANNOT_EMBED_CONFIGURATION defined(__AVR__) +#endif diff --git a/Marlin/src/inc/Warnings.cpp b/Marlin/src/inc/Warnings.cpp index b7eef9c49f..c2fe42ae82 100644 --- a/Marlin/src/inc/Warnings.cpp +++ b/Marlin/src/inc/Warnings.cpp @@ -549,3 +549,7 @@ #elif !USE_SENSORLESS && ENABLED(USES_DIAG_PINS) #warning "Driver DIAG pins must be physically removed unless SENSORLESS_HOMING is enabled. (See https://bit.ly/2ZPRlt0)" #endif + +#if CANNOT_EMBED_CONFIGURATION + #warning "Disabled CONFIGURATION_EMBEDDING because the target usually has less flash storage. Define FORCE_CONFIG_EMBED to override." +#endif diff --git a/Marlin/src/pins/stm32g0/pins_BTT_SKR_MINI_E3_V3_0.h b/Marlin/src/pins/stm32g0/pins_BTT_SKR_MINI_E3_V3_0.h index 2fda0fb2c0..cec1a838fc 100644 --- a/Marlin/src/pins/stm32g0/pins_BTT_SKR_MINI_E3_V3_0.h +++ b/Marlin/src/pins/stm32g0/pins_BTT_SKR_MINI_E3_V3_0.h @@ -150,7 +150,7 @@ * (LCD_D4) PB9 | 5 6 PA10 (BTN_EN2) * RESET | 7 8 | PA9 (BTN_EN1) * (BTN_ENC) PA15 | 9 10 | PB5 (BEEPER) - * ------ + * ------ * EXP1 */ #define EXP1_09_PIN PA15 diff --git a/buildroot/bin/restore_configs b/buildroot/bin/restore_configs index b2d0ea19ac..61aa3f9ee1 100755 --- a/buildroot/bin/restore_configs +++ b/buildroot/bin/restore_configs @@ -2,4 +2,4 @@ git checkout Marlin/Configuration*.h 2>/dev/null git checkout Marlin/src/pins/ramps/pins_RAMPS.h 2>/dev/null -rm -f Marlin/_Bootscreen.h Marlin/_Statusscreen.h +rm -f Marlin/_Bootscreen.h Marlin/_Statusscreen.h marlin_config.json .pio/build/mc.zip diff --git a/buildroot/share/PlatformIO/scripts/SAMD51_grandcentral_m4.py b/buildroot/share/PlatformIO/scripts/SAMD51_grandcentral_m4.py index be2c87266a..e7442f2485 100644 --- a/buildroot/share/PlatformIO/scripts/SAMD51_grandcentral_m4.py +++ b/buildroot/share/PlatformIO/scripts/SAMD51_grandcentral_m4.py @@ -6,7 +6,6 @@ import pioutil if pioutil.is_pio_build(): from os.path import join, isfile import shutil - from pprint import pprint Import("env") diff --git a/buildroot/share/PlatformIO/scripts/__init__.py b/buildroot/share/PlatformIO/scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/buildroot/share/PlatformIO/scripts/common-dependencies.py b/buildroot/share/PlatformIO/scripts/common-dependencies.py index 004f78f19b..24e780d9b6 100644 --- a/buildroot/share/PlatformIO/scripts/common-dependencies.py +++ b/buildroot/share/PlatformIO/scripts/common-dependencies.py @@ -192,63 +192,6 @@ if pioutil.is_pio_build(): lib_ignore = env.GetProjectOption('lib_ignore') + [feat['lib_ignore']] set_env_field('lib_ignore', lib_ignore) - # - # Find a compiler, considering the OS - # - ENV_BUILD_PATH = os.path.join(env.Dictionary('PROJECT_BUILD_DIR'), env['PIOENV']) - GCC_PATH_CACHE = os.path.join(ENV_BUILD_PATH, ".gcc_path") - def search_compiler(): - try: - filepath = env.GetProjectOption('custom_gcc') - blab("Getting compiler from env") - return filepath - except: - pass - - if os.path.exists(GCC_PATH_CACHE): - with open(GCC_PATH_CACHE, 'r') as f: - return f.read() - - # Find the current platform compiler by searching the $PATH - # which will be in a platformio toolchain bin folder - path_regex = re.escape(env['PROJECT_PACKAGES_DIR']) - - # See if the environment provides a default compiler - try: - gcc = env.GetProjectOption('custom_deps_gcc') - except: - gcc = "g++" - - if env['PLATFORM'] == 'win32': - path_separator = ';' - path_regex += r'.*\\bin' - gcc += ".exe" - else: - path_separator = ':' - path_regex += r'/.+/bin' - - # Search for the compiler - for pathdir in env['ENV']['PATH'].split(path_separator): - if not re.search(path_regex, pathdir, re.IGNORECASE): - continue - for filepath in os.listdir(pathdir): - if not filepath.endswith(gcc): - continue - # Use entire path to not rely on env PATH - filepath = os.path.sep.join([pathdir, filepath]) - # Cache the g++ path to no search always - if os.path.exists(ENV_BUILD_PATH): - with open(GCC_PATH_CACHE, 'w+') as f: - f.write(filepath) - - return filepath - - filepath = env.get('CXX') - if filepath == 'CC': - filepath = gcc - blab("Couldn't find a compiler! Fallback to %s" % filepath) - return filepath - # # Use the compiler to get a list of all enabled features # @@ -257,25 +200,8 @@ if pioutil.is_pio_build(): return # Process defines - build_flags = env.get('BUILD_FLAGS') - build_flags = env.ParseFlagsExtended(build_flags) - - cxx = search_compiler() - cmd = ['"' + cxx + '"'] - - # Build flags from board.json - #if 'BOARD' in env: - # cmd += [env.BoardConfig().get("build.extra_flags")] - for s in build_flags['CPPDEFINES']: - if isinstance(s, tuple): - cmd += ['-D' + s[0] + '=' + str(s[1])] - else: - cmd += ['-D' + s] - - cmd += ['-D__MARLIN_DEPS__ -w -dM -E -x c++ buildroot/share/PlatformIO/scripts/common-dependencies.h'] - cmd = ' '.join(cmd) - blab(cmd, 4) - define_list = subprocess.check_output(cmd, shell=True).splitlines() + from preprocessor import run_preprocessor + define_list = run_preprocessor(env) marlin_features = {} for define in define_list: feature = define[8:].strip().decode().split(' ') @@ -310,9 +236,18 @@ if pioutil.is_pio_build(): except: pass + # # Add a method for other PIO scripts to query enabled features + # env.AddMethod(MarlinFeatureIsEnabled) + # # Add dependencies for enabled Marlin features + # apply_features_config() force_ignore_unused_libs() + + #print(env.Dump()) + + from signature import compute_build_signature + compute_build_signature(env) diff --git a/buildroot/share/PlatformIO/scripts/mc-apply.py b/buildroot/share/PlatformIO/scripts/mc-apply.py new file mode 100755 index 0000000000..f71d192679 --- /dev/null +++ b/buildroot/share/PlatformIO/scripts/mc-apply.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# +# Create a Configuration from marlin_config.json +# +import json +import sys +import shutil +import re + +opt_output = '--opt' in sys.argv +output_suffix = '.sh' if opt_output else '' if '--bare-output' in sys.argv else '.gen' + +try: + with open('marlin_config.json', 'r') as infile: + conf = json.load(infile) + for key in conf: + # We don't care about the hash when restoring here + if key == '__INITIAL_HASH': + continue + if key == 'VERSION': + for k, v in sorted(conf[key].items()): + print(k + ': ' + v) + continue + # The key is the file name, so let's build it now + outfile = open('Marlin/' + key + output_suffix, 'w') + for k, v in sorted(conf[key].items()): + # Make define line now + if opt_output: + if v != '': + if '"' in v: + v = "'%s'" % v + elif ' ' in v: + v = '"%s"' % v + define = 'opt_set ' + k + ' ' + v + '\n' + else: + define = 'opt_enable ' + k + '\n' + else: + define = '#define ' + k + ' ' + v + '\n' + outfile.write(define) + outfile.close() + + # Try to apply changes to the actual configuration file (in order to keep useful comments) + if output_suffix != '': + # Move the existing configuration so it doesn't interfere + shutil.move('Marlin/' + key, 'Marlin/' + key + '.orig') + infile_lines = open('Marlin/' + key + '.orig', 'r').read().split('\n') + outfile = open('Marlin/' + key, 'w') + for line in infile_lines: + sline = line.strip(" \t\n\r") + if sline[:7] == "#define": + # Extract the key here (we don't care about the value) + kv = sline[8:].strip().split(' ') + if kv[0] in conf[key]: + outfile.write('#define ' + kv[0] + ' ' + conf[key][kv[0]] + '\n') + # Remove the key from the dict, so we can still write all missing keys at the end of the file + del conf[key][kv[0]] + else: + outfile.write(line + '\n') + else: + outfile.write(line + '\n') + # Process any remaining defines here + for k, v in sorted(conf[key].items()): + define = '#define ' + k + ' ' + v + '\n' + outfile.write(define) + outfile.close() + + print('Output configuration written to: ' + 'Marlin/' + key + output_suffix) +except: + print('No marlin_config.json found.') diff --git a/buildroot/share/PlatformIO/scripts/preprocessor.py b/buildroot/share/PlatformIO/scripts/preprocessor.py new file mode 100644 index 0000000000..6f4ed900ca --- /dev/null +++ b/buildroot/share/PlatformIO/scripts/preprocessor.py @@ -0,0 +1,99 @@ +# +# preprocessor.py +# +import subprocess,os,re + +verbose = 0 + +def blab(str): + if verbose: + print(str) + +################################################################################ +# +# Invoke GCC to run the preprocessor and extract enabled features +# +preprocessor_cache = {} +def run_preprocessor(env, fn=None): + filename = fn or 'buildroot/share/PlatformIO/scripts/common-dependencies.h' + if filename in preprocessor_cache: + return preprocessor_cache[filename] + + # Process defines + build_flags = env.get('BUILD_FLAGS') + build_flags = env.ParseFlagsExtended(build_flags) + + cxx = search_compiler(env) + cmd = ['"' + cxx + '"'] + + # Build flags from board.json + #if 'BOARD' in env: + # cmd += [env.BoardConfig().get("build.extra_flags")] + for s in build_flags['CPPDEFINES']: + if isinstance(s, tuple): + cmd += ['-D' + s[0] + '=' + str(s[1])] + else: + cmd += ['-D' + s] + + cmd += ['-D__MARLIN_DEPS__ -w -dM -E -x c++'] + depcmd = cmd + [ filename ] + cmd = ' '.join(depcmd) + blab(cmd) + define_list = subprocess.check_output(cmd, shell=True).splitlines() + preprocessor_cache[filename] = define_list + return define_list + + +################################################################################ +# +# Find a compiler, considering the OS +# +def search_compiler(env): + + ENV_BUILD_PATH = os.path.join(env.Dictionary('PROJECT_BUILD_DIR'), env['PIOENV']) + GCC_PATH_CACHE = os.path.join(ENV_BUILD_PATH, ".gcc_path") + + try: + filepath = env.GetProjectOption('custom_gcc') + blab("Getting compiler from env") + return filepath + except: + pass + + if os.path.exists(GCC_PATH_CACHE): + blab("Getting g++ path from cache") + with open(GCC_PATH_CACHE, 'r') as f: + return f.read() + + # Find the current platform compiler by searching the $PATH + # which will be in a platformio toolchain bin folder + path_regex = re.escape(env['PROJECT_PACKAGES_DIR']) + gcc = "g++" + if env['PLATFORM'] == 'win32': + path_separator = ';' + path_regex += r'.*\\bin' + gcc += ".exe" + else: + path_separator = ':' + path_regex += r'/.+/bin' + + # Search for the compiler + for pathdir in env['ENV']['PATH'].split(path_separator): + if not re.search(path_regex, pathdir, re.IGNORECASE): + continue + for filepath in os.listdir(pathdir): + if not filepath.endswith(gcc): + continue + # Use entire path to not rely on env PATH + filepath = os.path.sep.join([pathdir, filepath]) + # Cache the g++ path to no search always + if os.path.exists(ENV_BUILD_PATH): + blab("Caching g++ for current env") + with open(GCC_PATH_CACHE, 'w+') as f: + f.write(filepath) + + return filepath + + filepath = env.get('CXX') + blab("Couldn't find a compiler! Fallback to %s" % filepath) + return filepath diff --git a/buildroot/share/PlatformIO/scripts/signature.py b/buildroot/share/PlatformIO/scripts/signature.py new file mode 100644 index 0000000000..654e3ea677 --- /dev/null +++ b/buildroot/share/PlatformIO/scripts/signature.py @@ -0,0 +1,176 @@ +# +# signature.py +# +import os,subprocess,re,json,hashlib + +# +# The dumbest preprocessor in the world +# Extract macro name from an header file and store them in an array +# No processing is done here, so they are raw values here and it does not match what actually enabled +# in the file (since you can have #if SOMETHING_UNDEFINED / #define BOB / #endif) +# But it's useful to filter the useful macro spit out by the preprocessor from noise from the system +# headers. +# +def extract_defines(filepath): + f = open(filepath, encoding="utf8").read().split("\n") + a = [] + for line in f: + sline = line.strip(" \t\n\r") + if sline[:7] == "#define": + # Extract the key here (we don't care about the value) + kv = sline[8:].strip().split(' ') + a.append(kv[0]) + return a + +# Compute the SHA256 hash of a file +def get_file_sha256sum(filepath): + sha256_hash = hashlib.sha256() + with open(filepath,"rb") as f: + # Read and update hash string value in blocks of 4K + for byte_block in iter(lambda: f.read(4096),b""): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() + +# +# Compress a JSON file into a zip file +# +import zipfile +def compress_file(filepath, outputbase): + with zipfile.ZipFile(outputbase + '.zip', 'w', compression=zipfile.ZIP_BZIP2, compresslevel=9) as zipf: + zipf.write(filepath, compress_type=zipfile.ZIP_BZIP2, compresslevel=9) + +# +# Compute the build signature. The idea is to extract all defines in the configuration headers +# to build a unique reversible signature from this build so it can be included in the binary +# We can reverse the signature to get a 1:1 equivalent configuration file +# +def compute_build_signature(env): + if 'BUILD_SIGNATURE' in env: + return + + # Definitions from these files will be kept + files_to_keep = [ 'Marlin/Configuration.h', 'Marlin/Configuration_adv.h' ] + + build_dir=os.path.join(env['PROJECT_BUILD_DIR'], env['PIOENV']) + + # Check if we can skip processing + hashes = '' + for header in files_to_keep: + hashes += get_file_sha256sum(header)[0:10] + + marlin_json = os.path.join(build_dir, 'marlin_config.json') + marlin_zip = os.path.join(build_dir, 'mc') + + # Read existing config file + try: + with open(marlin_json, 'r') as infile: + conf = json.load(infile) + if conf['__INITIAL_HASH'] == hashes: + # Same configuration, skip recomputing the building signature + compress_file(marlin_json, marlin_zip) + return + except: + pass + + # Get enabled config options based on preprocessor + from preprocessor import run_preprocessor + complete_cfg = run_preprocessor(env) + + # Dumb #define extraction from the configuration files + real_defines = {} + all_defines = [] + for header in files_to_keep: + defines = extract_defines(header) + # To filter only the define we want + all_defines = all_defines + defines + # To remember from which file it cames from + real_defines[header.split('/')[-1]] = defines + + r = re.compile(r"\(+(\s*-*\s*_.*)\)+") + + # First step is to collect all valid macros + defines = {} + for line in complete_cfg: + + # Split the define from the value + key_val = line[8:].strip().decode().split(' ') + key, value = key_val[0], ' '.join(key_val[1:]) + + # Ignore values starting with two underscore, since it's low level + if len(key) > 2 and key[0:2] == "__" : + continue + # Ignore values containing a parenthesis (likely a function macro) + if '(' in key and ')' in key: + continue + + # Then filter dumb values + if r.match(value): + continue + + defines[key] = value if len(value) else "" + + if not 'CONFIGURATION_EMBEDDING' in defines: + return + + # Second step is to filter useless macro + resolved_defines = {} + for key in defines: + # Remove all boards now + if key[0:6] == "BOARD_" and key != "BOARD_INFO_NAME": + continue + # Remove all keys ending by "_NAME" as it does not make a difference to the configuration + if key[-5:] == "_NAME" and key != "CUSTOM_MACHINE_NAME": + continue + # Remove all keys ending by "_T_DECLARED" as it's a copy of not important system stuff + if key[-11:] == "_T_DECLARED": + continue + # Remove keys that are not in the #define list in the Configuration list + if not (key in all_defines) and key != "DETAILED_BUILD_VERSION" and key != "STRING_DISTRIBUTION_DATE": + continue + + # Don't be that smart guy here + resolved_defines[key] = defines[key] + + # Generate a build signature now + # We are making an object that's a bit more complex than a basic dictionary here + data = {} + data['__INITIAL_HASH'] = hashes + # First create a key for each header here + for header in real_defines: + data[header] = {} + + # Then populate the object where each key is going to (that's a O(N^2) algorithm here...) + for key in resolved_defines: + for header in real_defines: + if key in real_defines[header]: + data[header][key] = resolved_defines[key] + + # Append the source code version and date + data['VERSION'] = {} + data['VERSION']['DETAILED_BUILD_VERSION'] = resolved_defines['DETAILED_BUILD_VERSION'] + data['VERSION']['STRING_DISTRIBUTION_DATE'] = resolved_defines['STRING_DISTRIBUTION_DATE'] + try: + curver = subprocess.check_output(["git", "describe", "--match=NeVeRmAtCh", "--always"]).strip() + data['VERSION']['GIT_REF'] = curver.decode() + except: + pass + + with open(marlin_json, 'w') as outfile: + json.dump(data, outfile, separators=(',', ':')) + + # Compress the JSON file as much as we can + compress_file(marlin_json, marlin_zip) + + # Generate a C source file for storing this array + with open('Marlin/src/mczip.h','wb') as result_file: + result_file.write(b'#warning "Generated file \'mc.zip\' is embedded"\n') + result_file.write(b'const unsigned char mc_zip[] PROGMEM = {\n ') + count = 0 + for b in open(os.path.join(build_dir, 'mc.zip'), 'rb').read(): + result_file.write(b' 0x%02X,' % b) + count += 1 + if (count % 16 == 0): + result_file.write(b'\n ') + if (count % 16): + result_file.write(b'\n') + result_file.write(b'};\n') diff --git a/docs/ConfigEmbedding.md b/docs/ConfigEmbedding.md new file mode 100644 index 0000000000..ed4ea39eda --- /dev/null +++ b/docs/ConfigEmbedding.md @@ -0,0 +1,19 @@ +# Configuration Embedding + +Starting with version 2.0.9.3, Marlin automatically extracts the configuration used to generate the firmware and stores it in the firmware binary. This is enabled by defining `CONFIGURATION_EMBEDDING` in `Configuration_adv.h`. + +## How it's done +To create the embedded configuration, we do a compiler pass to process the Configuration files and extract all active options. The active options are parsed into key/value pairs, serialized to JSON format, and stored in a file called `marlin_config.json`, which also includes specific build information (like the git revision, the build date, and some version information. The JSON file is then compressed in a ZIP archive called `.pio/build/mc.zip` which is converted into a C array and stored in a C++ file called `mc.h` which is included in the build. + +## Extracting configurations from a Marlin binary +To get the configuration out of a binary firmware, you'll need a non-write-protected SD card inserted into the printer while running the firmware. +Send the command `M503 C` to write the file `mc.zip` to the SD card. Copy the file to your computer, ideally in the same folder as the Marlin repository. + +Run the following commands to extract and apply the configuration: +``` +$ git checkout -f +$ unzip mc.zip +$ python buildroot/share/PlatformIO/scripts/mc-apply.py +``` + +This will attempt to update the configuration files to match the settings used for the original build. It will also dump the git reference used to build the code (which may be accessible if the firmware was built from the main repository. As a fallback it also includes the `STRING_DISTRIBUTION_DATE` which is unlikely to be modified in a fork).